diff --git a/.changeset/ollama-instrumentation.md b/.changeset/ollama-instrumentation.md new file mode 100644 index 000000000..52443f3be --- /dev/null +++ b/.changeset/ollama-instrumentation.md @@ -0,0 +1,5 @@ +--- +"braintrust": minor +--- + +feat: Add `ollama` instrumentation diff --git a/AGENTS.md b/AGENTS.md index 610763e62..6338e8830 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,6 +57,8 @@ pnpm run test:e2e:record # Re-record provider cassettes and update snap When adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output. +New instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary. + Span-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair. **From repo root:** diff --git a/e2e/README.md b/e2e/README.md index c61436883..d9ac582ea 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -105,6 +105,7 @@ Provider credentials are only required when recording or explicitly running live - `CURSOR_API_KEY` - `OPENROUTER_API_KEY` - `MISTRAL_API_KEY` +- `OLLAMA_API_KEY` - `HUGGINGFACE_API_KEY` - `COHERE_API_KEY` - `GROQ_API_KEY` @@ -167,7 +168,7 @@ pnpm --filter=@braintrust/js-e2e-tests run test:e2e:record -- ANTHROPIC_API_KEY=... AWS_BEARER_TOKEN_BEDROCK=... \ OPENAI_API_KEY=... GEMINI_API_KEY=... \ COHERE_API_KEY=... GROQ_API_KEY=... HUGGINGFACE_API_KEY=... \ -MISTRAL_API_KEY=... OPENROUTER_API_KEY=... \ +MISTRAL_API_KEY=... OLLAMA_API_KEY=... OPENROUTER_API_KEY=... \ CURSOR_API_KEY=... \ pnpm --filter=@braintrust/js-e2e-tests run test:e2e:record ``` @@ -175,7 +176,7 @@ CURSOR_API_KEY=... \ After recording, run again **without any provider keys** to confirm the cassette is sufficient: ```bash -unset ANTHROPIC_API_KEY AWS_BEARER_TOKEN_BEDROCK OPENAI_API_KEY GEMINI_API_KEY GOOGLE_API_KEY GOOGLE_GENAI_API_KEY COHERE_API_KEY GROQ_API_KEY HUGGINGFACE_API_KEY MISTRAL_API_KEY OPENROUTER_API_KEY CURSOR_API_KEY +unset ANTHROPIC_API_KEY AWS_BEARER_TOKEN_BEDROCK OPENAI_API_KEY GEMINI_API_KEY GOOGLE_API_KEY GOOGLE_GENAI_API_KEY COHERE_API_KEY GROQ_API_KEY HUGGINGFACE_API_KEY MISTRAL_API_KEY OLLAMA_API_KEY OPENROUTER_API_KEY CURSOR_API_KEY pnpm --filter=@braintrust/js-e2e-tests run test:e2e ``` @@ -187,7 +188,7 @@ After any successful record run, stale cassette variants are cleaned only inside These scenarios have cassette wiring in place and will use cassettes once they're recorded: -`anthropic-bedrock-instrumentation`, `anthropic-instrumentation`, `openai-instrumentation`, `openai-codex-instrumentation`, `ai-sdk-instrumentation`, `ai-sdk-otel-export`, `claude-agent-sdk-instrumentation`, `cohere-instrumentation`, `cursor-sdk-instrumentation`, `github-copilot-instrumentation`, `google-adk-instrumentation`, `google-genai-instrumentation`, `groq-instrumentation`, `huggingface-instrumentation`, `mistral-instrumentation`, `openrouter-agent-instrumentation`, `openrouter-instrumentation`, `wrap-langchain-js-traces`. +`anthropic-bedrock-instrumentation`, `anthropic-instrumentation`, `openai-instrumentation`, `openai-codex-instrumentation`, `ai-sdk-instrumentation`, `ai-sdk-otel-export`, `claude-agent-sdk-instrumentation`, `cohere-instrumentation`, `cursor-sdk-instrumentation`, `github-copilot-instrumentation`, `google-adk-instrumentation`, `google-genai-instrumentation`, `groq-instrumentation`, `huggingface-instrumentation`, `mistral-instrumentation`, `ollama-instrumentation`, `openrouter-agent-instrumentation`, `openrouter-instrumentation`, `wrap-langchain-js-traces`. ### Cassette format diff --git a/e2e/config/pr-comment-scenarios.json b/e2e/config/pr-comment-scenarios.json index c4ad8be50..7cf0b43ad 100644 --- a/e2e/config/pr-comment-scenarios.json +++ b/e2e/config/pr-comment-scenarios.json @@ -412,6 +412,21 @@ } ] }, + { + "scenarioDirName": "ollama-instrumentation", + "label": "Ollama Instrumentation", + "metadataScenario": "ollama-instrumentation", + "variants": [ + { + "variantKey": "ollama-v0.6", + "label": "v0.6 pinned" + }, + { + "variantKey": "ollama-v0.6-latest", + "label": "v0.6 latest" + } + ] + }, { "scenarioDirName": "cohere-instrumentation", "label": "Cohere Instrumentation", diff --git a/e2e/helpers/scenario-harness.ts b/e2e/helpers/scenario-harness.ts index b41e3a8be..8390f5a87 100644 --- a/e2e/helpers/scenario-harness.ts +++ b/e2e/helpers/scenario-harness.ts @@ -315,6 +315,7 @@ function getCassetteServerRoutes(): CassetteServerRoute[] { upstreamOrigin: "https://router.huggingface.co", }, { prefix: "/mistral", upstreamOrigin: "https://api.mistral.ai" }, + { prefix: "/ollama", upstreamOrigin: "https://ollama.com" }, { prefix: "/openai", upstreamOrigin: "https://api.openai.com" }, { prefix: "/openrouter", upstreamOrigin: "https://openrouter.ai" }, ]; @@ -349,6 +350,7 @@ function getCassetteEnv(wiring: ActiveCassetteWiring): Record { HUGGINGFACE_ROUTER_BASE_URL: `${serverUrl}/huggingface-router`, MISTRAL_API_URL: `${serverUrl}/mistral`, MISTRAL_BASE_URL: `${serverUrl}/mistral`, + OLLAMA_HOST: `${serverUrl}/ollama`, OPENAI_BASE_URL: `${serverUrl}/openai/v1`, OPENROUTER_BASE_URL: `${serverUrl}/openrouter/api/v1`, }; @@ -398,6 +400,7 @@ const CASSETTE_PROVIDER_KEYS: Array<{ { envVars: ["GROQ_API_KEY"], placeholder: "gsk_cassette-placeholder" }, { envVars: ["HUGGINGFACE_API_KEY"], placeholder: "hf_cassette-placeholder" }, { envVars: ["MISTRAL_API_KEY"], placeholder: "cassette-placeholder" }, + { envVars: ["OLLAMA_API_KEY"], placeholder: "cassette-placeholder" }, { envVars: ["OPENAI_API_KEY"], placeholder: "sk-cassette-placeholder" }, { envVars: ["OPENROUTER_API_KEY"], diff --git a/e2e/scenarios/ollama-instrumentation/__cassettes__/ollama-v0.6-latest.cassette.json b/e2e/scenarios/ollama-instrumentation/__cassettes__/ollama-v0.6-latest.cassette.json new file mode 100644 index 000000000..f6c999c5e --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__cassettes__/ollama-v0.6-latest.cassette.json @@ -0,0 +1,228 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "2a86bcda6598aa82", + "matchKey": "POST ollama.com/api/chat", + "recordedAt": "2026-07-30T14:17:27.786Z", + "request": { + "body": { + "kind": "json", + "value": { + "messages": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "stream": false, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/chat" + }, + "response": { + "body": { + "kind": "json", + "value": { + "created_at": "2026-07-30T14:17:27.836768072Z", + "done": true, + "done_reason": "stop", + "eval_count": 40, + "message": { + "content": "OK", + "role": "assistant", + "thinking": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK." + }, + "model": "gpt-oss:20b", + "prompt_eval_count": 67, + "total_duration": 820667686 + } + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-length": "349", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:27 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-c66f99321115f5f107758a3fad47a00f-bcbe1f3d1e4384ae-00", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "c66f99321115f5f107758a3fad47a00f/13600342272066684078", + "x-frame-options": "DENY", + "x-request-id": "127f9275-9630-4810-9a7c-6b72c370cb3c" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 1, + "id": "923a3e6055148564", + "matchKey": "POST ollama.com/api/chat", + "recordedAt": "2026-07-30T14:17:29.278Z", + "request": { + "body": { + "kind": "json", + "value": { + "messages": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "stream": true, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/chat" + }, + "response": { + "body": { + "kind": "text", + "value": "{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.34967424Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"The\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.359378568Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" user\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.390259604Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" says\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.396460009Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\":\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.396601569Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.3975552Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"Reply\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.405247206Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" with\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.407624928Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" exactly\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.418610508Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.464847596Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.465359817Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" So\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.465435117Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" we\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.465480977Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" must\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.478837208Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" output\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.478898448Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" exactly\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.488134706Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" the\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.496465303Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" word\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.532802133Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.532881813Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.537893028Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"\\\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.547788406Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" No\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.555467652Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" extra\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.571374226Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" spaces\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.571476426Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\",\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.587682829Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" no\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.592180173Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" newline\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.593067434Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"?\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.606547985Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" They\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.615570543Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" said\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.645977628Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.646838859Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"Reply\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.655935567Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" with\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.665154714Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" exactly\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.673848022Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.689652135Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.689813995Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" Usually\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.702106365Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" we\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.710015642Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" output\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.717246658Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" the\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.726397266Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" word\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.735010853Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.789715719Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" Probably\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.790339829Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" just\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.790963009Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.79104392Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.79129428Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"\\\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.79150011Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" No\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.800693548Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" punctuation\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.807873884Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.808396504Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" So\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.819012393Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" output\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.840200251Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\":\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.840856441Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.843193773Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:29.248242423Z\",\"message\":{\"role\":\"assistant\",\"content\":\"STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:29.255514869Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:29.330188961Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\"},\"done\":true,\"done_reason\":\"stop\",\"total_duration\":1363219361,\"prompt_eval_count\":67,\"eval_count\":65}\n" + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:28 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-4455c46106092326278890284531f2ca-df664b0c9e5442e6-00", + "transfer-encoding": "chunked", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "4455c46106092326278890284531f2ca/16097636435650429670", + "x-frame-options": "DENY", + "x-request-id": "55fbd7c8-7512-4361-a3ae-8f2810927893" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 0, + "id": "99f1d8c4b0c43219", + "matchKey": "POST ollama.com/api/generate", + "recordedAt": "2026-07-30T14:17:33.894Z", + "request": { + "body": { + "kind": "json", + "value": { + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "prompt": "Reply with exactly OK.", + "stream": false, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/generate" + }, + "response": { + "body": { + "kind": "json", + "value": { + "created_at": "2026-07-30T14:17:33.941708086Z", + "done": true, + "done_reason": "stop", + "eval_count": 40, + "model": "gpt-oss:20b", + "prompt_eval_count": 67, + "response": "OK", + "thinking": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "total_duration": 4489076853 + } + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-length": "320", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:33 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-106b492de1a5a0ce278890284531f004-f53ba4690e6cf0bc-00", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "106b492de1a5a0ce278890284531f004/17670898334038814908", + "x-frame-options": "DENY", + "x-request-id": "203de0b5-0587-4e63-88c6-e083c1074a17" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 1, + "id": "54471dd63ef97520", + "matchKey": "POST ollama.com/api/generate", + "recordedAt": "2026-07-30T14:17:34.958Z", + "request": { + "body": { + "kind": "json", + "value": { + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "prompt": "Reply with exactly STREAM.", + "stream": true, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/generate" + }, + "response": { + "body": { + "kind": "text", + "value": "{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.389627377Z\",\"response\":\"\",\"thinking\":\"The\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.395533242Z\",\"response\":\"\",\"thinking\":\" user\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.403797509Z\",\"response\":\"\",\"thinking\":\" says\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.412127386Z\",\"response\":\"\",\"thinking\":\":\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.420263983Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.431670083Z\",\"response\":\"\",\"thinking\":\"Reply\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.439038379Z\",\"response\":\"\",\"thinking\":\" with\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.445798215Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.453676261Z\",\"response\":\"\",\"thinking\":\" STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.462879529Z\",\"response\":\"\",\"thinking\":\".\\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.48754039Z\",\"response\":\"\",\"thinking\":\" So\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.497012667Z\",\"response\":\"\",\"thinking\":\" we\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.503352273Z\",\"response\":\"\",\"thinking\":\" must\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.508533877Z\",\"response\":\"\",\"thinking\":\" output\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.518284895Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.524960831Z\",\"response\":\"\",\"thinking\":\" the\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.532656867Z\",\"response\":\"\",\"thinking\":\" word\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.555859397Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.563573723Z\",\"response\":\"\",\"thinking\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.564276844Z\",\"response\":\"\",\"thinking\":\"\\\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.566337645Z\",\"response\":\"\",\"thinking\":\" No\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.573933902Z\",\"response\":\"\",\"thinking\":\" extra\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.581418518Z\",\"response\":\"\",\"thinking\":\" spaces\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.589888165Z\",\"response\":\"\",\"thinking\":\",\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.601848775Z\",\"response\":\"\",\"thinking\":\" no\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.610821713Z\",\"response\":\"\",\"thinking\":\" newline\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.615298726Z\",\"response\":\"\",\"thinking\":\"?\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.624505524Z\",\"response\":\"\",\"thinking\":\" They\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.632406001Z\",\"response\":\"\",\"thinking\":\" said\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.642595159Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.650027446Z\",\"response\":\"\",\"thinking\":\"Reply\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.661598875Z\",\"response\":\"\",\"thinking\":\" with\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.669144222Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.675096107Z\",\"response\":\"\",\"thinking\":\" STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.686189736Z\",\"response\":\"\",\"thinking\":\".\\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.692248381Z\",\"response\":\"\",\"thinking\":\" Usually\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.705089872Z\",\"response\":\"\",\"thinking\":\" we\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.709504965Z\",\"response\":\"\",\"thinking\":\" output\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.720336364Z\",\"response\":\"\",\"thinking\":\" the\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.72721519Z\",\"response\":\"\",\"thinking\":\" word\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.735638087Z\",\"response\":\"\",\"thinking\":\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.747191607Z\",\"response\":\"\",\"thinking\":\" Probably\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.760642398Z\",\"response\":\"\",\"thinking\":\" just\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.778758373Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.779113164Z\",\"response\":\"\",\"thinking\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.798139999Z\",\"response\":\"\",\"thinking\":\"\\\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.799441891Z\",\"response\":\"\",\"thinking\":\" No\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.799505571Z\",\"response\":\"\",\"thinking\":\" punctuation\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.808817408Z\",\"response\":\"\",\"thinking\":\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.827693204Z\",\"response\":\"\",\"thinking\":\" So\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.837738383Z\",\"response\":\"\",\"thinking\":\" final\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.849834023Z\",\"response\":\"\",\"thinking\":\" answer\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.856697219Z\",\"response\":\"\",\"thinking\":\":\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.863521814Z\",\"response\":\"\",\"thinking\":\" STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.865494336Z\",\"response\":\"\",\"thinking\":\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.927466028Z\",\"response\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.937183716Z\",\"response\":\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:35.009953287Z\",\"response\":\"\",\"done\":true,\"done_reason\":\"stop\",\"total_duration\":946827533,\"prompt_eval_count\":67,\"eval_count\":66}\n" + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:34 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-e8e48c6fbc8884675481ff7328ae2d0b-7a094f57d5536a16-00", + "transfer-encoding": "chunked", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "e8e48c6fbc8884675481ff7328ae2d0b/8793646986077366806", + "x-frame-options": "DENY", + "x-request-id": "97ff230e-37fe-4bb0-a698-396b552dcdcb" + }, + "status": 200, + "statusText": "OK" + } + } + ], + "meta": { + "createdAt": "2026-07-30T14:15:13.600Z" + } +} diff --git a/e2e/scenarios/ollama-instrumentation/__cassettes__/ollama-v0.6.cassette.json b/e2e/scenarios/ollama-instrumentation/__cassettes__/ollama-v0.6.cassette.json new file mode 100644 index 000000000..e092efa99 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__cassettes__/ollama-v0.6.cassette.json @@ -0,0 +1,228 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "2a86bcda6598aa82", + "matchKey": "POST ollama.com/api/chat", + "recordedAt": "2026-07-30T14:17:15.919Z", + "request": { + "body": { + "kind": "json", + "value": { + "messages": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "stream": false, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/chat" + }, + "response": { + "body": { + "kind": "json", + "value": { + "created_at": "2026-07-30T14:17:15.968759871Z", + "done": true, + "done_reason": "stop", + "eval_count": 40, + "message": { + "content": "OK", + "role": "assistant", + "thinking": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK." + }, + "model": "gpt-oss:20b", + "prompt_eval_count": 67, + "total_duration": 2759051201 + } + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-length": "350", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:15 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-499b9cbcd8010b751b766ba772b23e2e-45c927d9e2a30383-00", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "499b9cbcd8010b751b766ba772b23e2e/5028594275699590019", + "x-frame-options": "DENY", + "x-request-id": "69feef07-e84b-461c-b965-82ca63d5cd59" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 1, + "id": "923a3e6055148564", + "matchKey": "POST ollama.com/api/chat", + "recordedAt": "2026-07-30T14:17:16.947Z", + "request": { + "body": { + "kind": "json", + "value": { + "messages": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "stream": true, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/chat" + }, + "response": { + "body": { + "kind": "text", + "value": "{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.420595067Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"The\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.428180554Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" user\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.434645149Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" says\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.443487256Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\":\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.454888486Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.464434994Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"Reply\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.468907788Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" with\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.475863774Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" exactly\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.486244792Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.494834779Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.501858345Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" So\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.513830845Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" the\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.522514833Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" assistant\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.528664688Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" should\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.536021444Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" reply\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.54387051Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" with\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.553723509Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" exactly\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.561350065Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" the\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.588566508Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" word\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.598017396Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.60322265Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.609424945Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"\\\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.619468064Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" No\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.643468554Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" other\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.644496975Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" characters\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.65059612Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\",\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.665049332Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" no\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.666145933Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" punctuation\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.675696741Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\",\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.685230109Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" no\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.68602731Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" whitespace\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.702263893Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.718099466Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" Just\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.764431305Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.794734681Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.843230121Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"\\\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.895195035Z\",\"message\":{\"role\":\"assistant\",\"content\":\"STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.897310446Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.996437479Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\"},\"done\":true,\"done_reason\":\"stop\",\"total_duration\":913140194,\"prompt_eval_count\":67,\"eval_count\":47}\n" + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:16 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-222faed9a3a0e3c91b766ba772b23075-d9f7f454c56193bb-00", + "transfer-encoding": "chunked", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "222faed9a3a0e3c91b766ba772b23075/15706290870403830715", + "x-frame-options": "DENY", + "x-request-id": "56b16227-d494-42bb-abf8-9db1b40d9f6d" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 0, + "id": "99f1d8c4b0c43219", + "matchKey": "POST ollama.com/api/generate", + "recordedAt": "2026-07-30T14:17:18.001Z", + "request": { + "body": { + "kind": "json", + "value": { + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "prompt": "Reply with exactly OK.", + "stream": false, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/generate" + }, + "response": { + "body": { + "kind": "json", + "value": { + "created_at": "2026-07-30T14:17:18.050054605Z", + "done": true, + "done_reason": "stop", + "eval_count": 40, + "model": "gpt-oss:20b", + "prompt_eval_count": 67, + "response": "OK", + "thinking": "The user says: \"Reply with exactly OK.\" So we must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "total_duration": 947970164 + } + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-length": "320", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:18 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-163ade5657ad33301b766ba772b2374e-25ddb75508a07d45-00", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "163ade5657ad33301b766ba772b2374e/2728538525101227333", + "x-frame-options": "DENY", + "x-request-id": "ccf7a2c3-d525-44ad-b0fa-cfa55a512239" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 1, + "id": "54471dd63ef97520", + "matchKey": "POST ollama.com/api/generate", + "recordedAt": "2026-07-30T14:17:19.889Z", + "request": { + "body": { + "kind": "json", + "value": { + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "prompt": "Reply with exactly STREAM.", + "stream": true, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/generate" + }, + "response": { + "body": { + "kind": "text", + "value": "{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.599724432Z\",\"response\":\"\",\"thinking\":\"The\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.601082173Z\",\"response\":\"\",\"thinking\":\" user\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.601825504Z\",\"response\":\"\",\"thinking\":\" says\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.603175275Z\",\"response\":\"\",\"thinking\":\":\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.604566096Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.605298347Z\",\"response\":\"\",\"thinking\":\"Reply\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.605716007Z\",\"response\":\"\",\"thinking\":\" with\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657090971Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657144751Z\",\"response\":\"\",\"thinking\":\" STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.65718348Z\",\"response\":\"\",\"thinking\":\".\\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657229391Z\",\"response\":\"\",\"thinking\":\" So\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657271011Z\",\"response\":\"\",\"thinking\":\" the\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657398341Z\",\"response\":\"\",\"thinking\":\" assistant\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657473381Z\",\"response\":\"\",\"thinking\":\" should\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657507981Z\",\"response\":\"\",\"thinking\":\" reply\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657562171Z\",\"response\":\"\",\"thinking\":\" with\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657608451Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657649941Z\",\"response\":\"\",\"thinking\":\" the\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657975201Z\",\"response\":\"\",\"thinking\":\" word\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658022951Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658079621Z\",\"response\":\"\",\"thinking\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658124381Z\",\"response\":\"\",\"thinking\":\"\\\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658174071Z\",\"response\":\"\",\"thinking\":\" No\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658224241Z\",\"response\":\"\",\"thinking\":\" other\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658275511Z\",\"response\":\"\",\"thinking\":\" characters\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658624692Z\",\"response\":\"\",\"thinking\":\",\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658671932Z\",\"response\":\"\",\"thinking\":\" no\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658725682Z\",\"response\":\"\",\"thinking\":\" punctuation\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658773082Z\",\"response\":\"\",\"thinking\":\",\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.720832694Z\",\"response\":\"\",\"thinking\":\" no\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.721380494Z\",\"response\":\"\",\"thinking\":\" whitespace\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.721458554Z\",\"response\":\"\",\"thinking\":\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.721841315Z\",\"response\":\"\",\"thinking\":\" Just\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.721914635Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.721965435Z\",\"response\":\"\",\"thinking\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722044485Z\",\"response\":\"\",\"thinking\":\"\\\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722130975Z\",\"response\":\"\",\"thinking\":\" Ensure\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722193605Z\",\"response\":\"\",\"thinking\":\" no\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722236845Z\",\"response\":\"\",\"thinking\":\" trailing\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722295275Z\",\"response\":\"\",\"thinking\":\" spaces\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722455815Z\",\"response\":\"\",\"thinking\":\" or\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722508865Z\",\"response\":\"\",\"thinking\":\" new\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722556475Z\",\"response\":\"\",\"thinking\":\"lines\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722604435Z\",\"response\":\"\",\"thinking\":\"?\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722650915Z\",\"response\":\"\",\"thinking\":\" The\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.723595826Z\",\"response\":\"\",\"thinking\":\" instruction\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.723652066Z\",\"response\":\"\",\"thinking\":\":\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.723690436Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.723731326Z\",\"response\":\"\",\"thinking\":\"Reply\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.723800286Z\",\"response\":\"\",\"thinking\":\" with\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.723847616Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.724054236Z\",\"response\":\"\",\"thinking\":\" STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.840079094Z\",\"response\":\"\",\"thinking\":\".\\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.840384084Z\",\"response\":\"\",\"thinking\":\" Usually\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.840527404Z\",\"response\":\"\",\"thinking\":\" means\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.841446975Z\",\"response\":\"\",\"thinking\":\" output\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.842998126Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.843058326Z\",\"response\":\"\",\"thinking\":\" the\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.843340866Z\",\"response\":\"\",\"thinking\":\" word\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.843399236Z\",\"response\":\"\",\"thinking\":\" STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.844343077Z\",\"response\":\"\",\"thinking\":\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.844413087Z\",\"response\":\"\",\"thinking\":\" So\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.846992499Z\",\"response\":\"\",\"thinking\":\" I\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.84708117Z\",\"response\":\"\",\"thinking\":\" will\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.848529051Z\",\"response\":\"\",\"thinking\":\" output\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.849822492Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.851341573Z\",\"response\":\"\",\"thinking\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.851916643Z\",\"response\":\"\",\"thinking\":\"\\\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.85971428Z\",\"response\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.85989297Z\",\"response\":\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.937042165Z\",\"response\":\"\",\"done\":true,\"done_reason\":\"stop\",\"total_duration\":1721668042,\"prompt_eval_count\":67,\"eval_count\":79}\n" + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:19 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-55d003c552c6028927ca4a55359c9d81-3b8fecdadd2af271-00", + "transfer-encoding": "chunked", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "55d003c552c6028927ca4a55359c9d81/4291909394664977009", + "x-frame-options": "DENY", + "x-request-id": "41903295-2fcb-4eb9-a612-8789e04d7cd7" + }, + "status": 200, + "statusText": "OK" + } + } + ], + "meta": { + "createdAt": "2026-07-30T13:54:11.198Z" + } +} diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.json b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.json new file mode 100644 index 000000000..c0779a0ce --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.json @@ -0,0 +1,307 @@ +{ + "span_tree": [ + { + "name": "ollama-instrumentation-root", + "type": "task", + "children": [ + { + "name": "ollama-chat-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "ollama-chat-stream-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\".", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 47, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 114 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "ollama-generate-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So we must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "ollama-generate-stream-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\". Ensure no trailing spaces or newlines? The instruction: \"Reply with exactly STREAM.\" Usually means output exactly the word STREAM. So I will output \"STREAM\".", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 79, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 146 + } + } + ], + "metadata": { + "operation": "generate-stream", + "testRunId": "" + } + }, + { + "name": "ollama-tool-call-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use get_temperature for Paris.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Paris\"}", + "name": "get_temperature" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama", + "tools": [ + { + "function": { + "description": "Get the temperature for a city", + "name": "get_temperature", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 5, + "prompt_tokens": 12, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "tool-call", + "testRunId": "" + } + }, + { + "name": "ollama-embed-operation", + "children": [ + { + "name": "ollama.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing", + "ollama instrumentation" + ], + "output": { + "embedding_length": 4 + }, + "metadata": { + "model": "embeddinggemma", + "provider": "ollama" + }, + "metrics": { + "prompt_tokens": 4, + "tokens": 4 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "ollama-error-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Trigger an error.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama" + }, + "error": "synthetic Ollama failure" + } + ], + "metadata": { + "operation": "error", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.txt b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.txt new file mode 100644 index 000000000..fe6c0d5f8 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.txt @@ -0,0 +1,243 @@ +span_tree: +└── ollama-instrumentation-root [task] + metadata: { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + ├── ollama-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\".", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 47, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 114 + │ } + ├── ollama-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So we must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-generate-stream-operation + │ metadata: { + │ "operation": "generate-stream", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\". Ensure no trailing spaces or newlines? The instruction: \"Reply with exactly STREAM.\" Usually means output exactly the word STREAM. So I will output \"STREAM\".", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 79, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 146 + │ } + ├── ollama-tool-call-operation + │ metadata: { + │ "operation": "tool-call", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Use get_temperature for Paris.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"city\":\"Paris\"}", + │ "name": "get_temperature" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the temperature for a city", + │ "name": "get_temperature", + │ "parameters": { + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_tokens": 12, + │ "tokens": 17 + │ } + ├── ollama-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── ollama.embed [llm] + │ input: [ + │ "braintrust tracing", + │ "ollama instrumentation" + │ ] + │ output: { + │ "embedding_length": 4 + │ } + │ metadata: { + │ "model": "embeddinggemma", + │ "provider": "ollama" + │ } + │ metrics: { + │ "prompt_tokens": 4, + │ "tokens": 4 + │ } + └── ollama-error-operation + metadata: { + "operation": "error", + "testRunId": "" + } + └── ollama.chat [llm] + input: [ + { + "content": "Trigger an error.", + "role": "user" + } + ] + metadata: { + "model": "gpt-oss:20b", + "provider": "ollama" + } + error: "synthetic Ollama failure" diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.json b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.json new file mode 100644 index 000000000..cf9cd619c --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.json @@ -0,0 +1,307 @@ +{ + "span_tree": [ + { + "name": "ollama-instrumentation-root", + "type": "task", + "children": [ + { + "name": "ollama-chat-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "ollama-chat-stream-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So output: STREAM.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 65, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 132 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "ollama-generate-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "ollama-generate-stream-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So final answer: STREAM.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 66, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 133 + } + } + ], + "metadata": { + "operation": "generate-stream", + "testRunId": "" + } + }, + { + "name": "ollama-tool-call-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use get_temperature for Paris.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Paris\"}", + "name": "get_temperature" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama", + "tools": [ + { + "function": { + "description": "Get the temperature for a city", + "name": "get_temperature", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 5, + "prompt_tokens": 12, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "tool-call", + "testRunId": "" + } + }, + { + "name": "ollama-embed-operation", + "children": [ + { + "name": "ollama.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing", + "ollama instrumentation" + ], + "output": { + "embedding_length": 4 + }, + "metadata": { + "model": "embeddinggemma", + "provider": "ollama" + }, + "metrics": { + "prompt_tokens": 4, + "tokens": 4 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "ollama-error-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Trigger an error.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama" + }, + "error": "synthetic Ollama failure" + } + ], + "metadata": { + "operation": "error", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.txt b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.txt new file mode 100644 index 000000000..8560418e2 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.txt @@ -0,0 +1,243 @@ +span_tree: +└── ollama-instrumentation-root [task] + metadata: { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + ├── ollama-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So output: STREAM.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 65, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 132 + │ } + ├── ollama-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-generate-stream-operation + │ metadata: { + │ "operation": "generate-stream", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So final answer: STREAM.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 66, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 133 + │ } + ├── ollama-tool-call-operation + │ metadata: { + │ "operation": "tool-call", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Use get_temperature for Paris.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"city\":\"Paris\"}", + │ "name": "get_temperature" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the temperature for a city", + │ "name": "get_temperature", + │ "parameters": { + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_tokens": 12, + │ "tokens": 17 + │ } + ├── ollama-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── ollama.embed [llm] + │ input: [ + │ "braintrust tracing", + │ "ollama instrumentation" + │ ] + │ output: { + │ "embedding_length": 4 + │ } + │ metadata: { + │ "model": "embeddinggemma", + │ "provider": "ollama" + │ } + │ metrics: { + │ "prompt_tokens": 4, + │ "tokens": 4 + │ } + └── ollama-error-operation + metadata: { + "operation": "error", + "testRunId": "" + } + └── ollama.chat [llm] + input: [ + { + "content": "Trigger an error.", + "role": "user" + } + ] + metadata: { + "model": "gpt-oss:20b", + "provider": "ollama" + } + error: "synthetic Ollama failure" diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.json b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.json new file mode 100644 index 000000000..cf9cd619c --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.json @@ -0,0 +1,307 @@ +{ + "span_tree": [ + { + "name": "ollama-instrumentation-root", + "type": "task", + "children": [ + { + "name": "ollama-chat-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "ollama-chat-stream-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So output: STREAM.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 65, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 132 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "ollama-generate-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "ollama-generate-stream-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So final answer: STREAM.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 66, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 133 + } + } + ], + "metadata": { + "operation": "generate-stream", + "testRunId": "" + } + }, + { + "name": "ollama-tool-call-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use get_temperature for Paris.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Paris\"}", + "name": "get_temperature" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama", + "tools": [ + { + "function": { + "description": "Get the temperature for a city", + "name": "get_temperature", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 5, + "prompt_tokens": 12, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "tool-call", + "testRunId": "" + } + }, + { + "name": "ollama-embed-operation", + "children": [ + { + "name": "ollama.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing", + "ollama instrumentation" + ], + "output": { + "embedding_length": 4 + }, + "metadata": { + "model": "embeddinggemma", + "provider": "ollama" + }, + "metrics": { + "prompt_tokens": 4, + "tokens": 4 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "ollama-error-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Trigger an error.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama" + }, + "error": "synthetic Ollama failure" + } + ], + "metadata": { + "operation": "error", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.txt b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.txt new file mode 100644 index 000000000..8560418e2 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.txt @@ -0,0 +1,243 @@ +span_tree: +└── ollama-instrumentation-root [task] + metadata: { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + ├── ollama-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So output: STREAM.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 65, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 132 + │ } + ├── ollama-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-generate-stream-operation + │ metadata: { + │ "operation": "generate-stream", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So final answer: STREAM.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 66, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 133 + │ } + ├── ollama-tool-call-operation + │ metadata: { + │ "operation": "tool-call", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Use get_temperature for Paris.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"city\":\"Paris\"}", + │ "name": "get_temperature" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the temperature for a city", + │ "name": "get_temperature", + │ "parameters": { + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_tokens": 12, + │ "tokens": 17 + │ } + ├── ollama-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── ollama.embed [llm] + │ input: [ + │ "braintrust tracing", + │ "ollama instrumentation" + │ ] + │ output: { + │ "embedding_length": 4 + │ } + │ metadata: { + │ "model": "embeddinggemma", + │ "provider": "ollama" + │ } + │ metrics: { + │ "prompt_tokens": 4, + │ "tokens": 4 + │ } + └── ollama-error-operation + metadata: { + "operation": "error", + "testRunId": "" + } + └── ollama.chat [llm] + input: [ + { + "content": "Trigger an error.", + "role": "user" + } + ] + metadata: { + "model": "gpt-oss:20b", + "provider": "ollama" + } + error: "synthetic Ollama failure" diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.json b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.json new file mode 100644 index 000000000..c0779a0ce --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.json @@ -0,0 +1,307 @@ +{ + "span_tree": [ + { + "name": "ollama-instrumentation-root", + "type": "task", + "children": [ + { + "name": "ollama-chat-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "ollama-chat-stream-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\".", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 47, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 114 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "ollama-generate-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So we must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "ollama-generate-stream-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\". Ensure no trailing spaces or newlines? The instruction: \"Reply with exactly STREAM.\" Usually means output exactly the word STREAM. So I will output \"STREAM\".", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 79, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 146 + } + } + ], + "metadata": { + "operation": "generate-stream", + "testRunId": "" + } + }, + { + "name": "ollama-tool-call-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use get_temperature for Paris.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Paris\"}", + "name": "get_temperature" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama", + "tools": [ + { + "function": { + "description": "Get the temperature for a city", + "name": "get_temperature", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 5, + "prompt_tokens": 12, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "tool-call", + "testRunId": "" + } + }, + { + "name": "ollama-embed-operation", + "children": [ + { + "name": "ollama.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing", + "ollama instrumentation" + ], + "output": { + "embedding_length": 4 + }, + "metadata": { + "model": "embeddinggemma", + "provider": "ollama" + }, + "metrics": { + "prompt_tokens": 4, + "tokens": 4 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "ollama-error-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Trigger an error.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama" + }, + "error": "synthetic Ollama failure" + } + ], + "metadata": { + "operation": "error", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.txt b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.txt new file mode 100644 index 000000000..fe6c0d5f8 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.txt @@ -0,0 +1,243 @@ +span_tree: +└── ollama-instrumentation-root [task] + metadata: { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + ├── ollama-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\".", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 47, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 114 + │ } + ├── ollama-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So we must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-generate-stream-operation + │ metadata: { + │ "operation": "generate-stream", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\". Ensure no trailing spaces or newlines? The instruction: \"Reply with exactly STREAM.\" Usually means output exactly the word STREAM. So I will output \"STREAM\".", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 79, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 146 + │ } + ├── ollama-tool-call-operation + │ metadata: { + │ "operation": "tool-call", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Use get_temperature for Paris.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"city\":\"Paris\"}", + │ "name": "get_temperature" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the temperature for a city", + │ "name": "get_temperature", + │ "parameters": { + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_tokens": 12, + │ "tokens": 17 + │ } + ├── ollama-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── ollama.embed [llm] + │ input: [ + │ "braintrust tracing", + │ "ollama instrumentation" + │ ] + │ output: { + │ "embedding_length": 4 + │ } + │ metadata: { + │ "model": "embeddinggemma", + │ "provider": "ollama" + │ } + │ metrics: { + │ "prompt_tokens": 4, + │ "tokens": 4 + │ } + └── ollama-error-operation + metadata: { + "operation": "error", + "testRunId": "" + } + └── ollama.chat [llm] + input: [ + { + "content": "Trigger an error.", + "role": "user" + } + ] + metadata: { + "model": "gpt-oss:20b", + "provider": "ollama" + } + error: "synthetic Ollama failure" diff --git a/e2e/scenarios/ollama-instrumentation/assertions.ts b/e2e/scenarios/ollama-instrumentation/assertions.ts new file mode 100644 index 000000000..a8dcd1df7 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/assertions.ts @@ -0,0 +1,214 @@ +import { beforeAll, describe, expect, test } from "vitest"; +import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; +import { + withScenarioHarness, + type ScenarioRunContext, +} from "../../helpers/scenario-harness"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; +import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; +import { + EMBEDDING_MODEL, + GENERATION_MODEL, + ROOT_NAME, + SCENARIO_NAME, +} from "./constants.mjs"; + +type RunOllamaScenario = (harness: { + runNodeScenarioDir: (options: { + entry: string; + nodeArgs: string[]; + runContext?: ScenarioRunContext; + scenarioDir: string; + timeoutMs: number; + }) => Promise; + runScenarioDir: (options: { + entry: string; + runContext?: ScenarioRunContext; + scenarioDir: string; + timeoutMs: number; + }) => Promise; +}) => Promise; + +function findProviderSpan( + events: CapturedLogEvent[], + operationName: string, + providerSpanName: string, +): CapturedLogEvent | undefined { + const operation = findLatestSpan(events, operationName); + const spans = findChildSpans(events, providerSpanName, operation?.span.id); + return ( + spans.find( + (candidate) => + candidate.output !== undefined || candidate.row.error !== undefined, + ) ?? spans[0] + ); +} + +function selectedEvents(events: CapturedLogEvent[]): CapturedLogEvent[] { + const operations = [ + ["ollama-chat-operation", "ollama.chat"], + ["ollama-chat-stream-operation", "ollama.chat"], + ["ollama-generate-operation", "ollama.generate"], + ["ollama-generate-stream-operation", "ollama.generate"], + ["ollama-tool-call-operation", "ollama.chat"], + ["ollama-embed-operation", "ollama.embed"], + ["ollama-error-operation", "ollama.chat"], + ] as const; + return [ + findLatestSpan(events, ROOT_NAME), + ...operations.flatMap(([operationName, providerSpanName]) => [ + findLatestSpan(events, operationName), + findProviderSpan(events, operationName, providerSpanName), + ]), + ].filter((event): event is CapturedLogEvent => event !== undefined); +} + +function expectTokenMetrics(event: CapturedLogEvent | undefined): void { + expect(event?.metrics).toMatchObject({ + prompt_tokens: expect.any(Number), + completion_tokens: expect.any(Number), + tokens: expect.any(Number), + }); +} + +export function defineOllamaInstrumentationAssertions(options: { + name: string; + runScenario: RunOllamaScenario; + snapshotName: string; + testFileUrl: string; + timeoutMs: number; +}): void { + const snapshotPath = resolveFileSnapshotPath( + options.testFileUrl, + `${options.snapshotName}.span-tree.json`, + ); + + describe(options.name, () => { + let events: CapturedLogEvent[] = []; + + beforeAll(async () => { + await withScenarioHarness(async (harness) => { + await options.runScenario(harness); + events = harness.events(); + }); + }, options.timeoutMs); + + test("captures chat and generate calls with streaming parity", () => { + for (const [operationName, spanName, streaming] of [ + ["ollama-chat-operation", "ollama.chat", false], + ["ollama-chat-stream-operation", "ollama.chat", true], + ["ollama-generate-operation", "ollama.generate", false], + ["ollama-generate-stream-operation", "ollama.generate", true], + ] as const) { + const span = findProviderSpan(events, operationName, spanName); + expect(span?.span.type).toBe("llm"); + expect(span?.row.metadata).toMatchObject({ + model: GENERATION_MODEL, + provider: "ollama", + }); + expect(span?.output).toEqual([ + expect.objectContaining({ + index: 0, + finish_reason: expect.any(String), + message: expect.objectContaining({ + role: "assistant", + content: expect.any(String), + }), + }), + ]); + expectTokenMetrics(span); + if (streaming) { + expect(span?.metrics?.time_to_first_token).toEqual( + expect.any(Number), + ); + } + expect(span?.row.context).toMatchObject({ + span_origin: { + instrumentation: { name: "ollama" }, + }, + }); + } + }); + + test("captures canonical tool definitions and calls", () => { + const span = findProviderSpan( + events, + "ollama-tool-call-operation", + "ollama.chat", + ); + const metadata = span?.row.metadata as + | Record + | undefined; + const output = span?.output as + | Array<{ + finish_reason?: string; + message?: { + tool_calls?: Array<{ + id?: string; + type?: string; + function?: { name?: string; arguments?: string }; + }>; + }; + }> + | undefined; + + expect(metadata?.tools).toEqual([ + { + type: "function", + function: { + name: "get_temperature", + description: "Get the temperature for a city", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }, + }, + }, + ]); + expect(output?.[0]?.finish_reason).toBe("tool_calls"); + expect(output?.[0]?.message?.tool_calls?.[0]).toMatchObject({ + id: "ollama_call_get_temperature_0", + type: "function", + function: { + name: "get_temperature", + arguments: '{"city":"Paris"}', + }, + }); + }); + + test("captures embedding calls compactly", () => { + const embed = findProviderSpan( + events, + "ollama-embed-operation", + "ollama.embed", + ); + + expect(embed?.row.metadata).toMatchObject({ + model: EMBEDDING_MODEL, + provider: "ollama", + }); + expect(embed?.output).toEqual({ embedding_length: 4 }); + expect(embed?.metrics).toMatchObject({ + prompt_tokens: 4, + tokens: 4, + }); + }); + + test("captures provider errors without changing caller behavior", () => { + const span = findProviderSpan( + events, + "ollama-error-operation", + "ollama.chat", + ); + expect(span?.row.error).toBe("synthetic Ollama failure"); + }); + + test("captures the scenario root and matches the span tree", async () => { + const root = findLatestSpan(events, ROOT_NAME); + expect(root?.row.metadata).toMatchObject({ scenario: SCENARIO_NAME }); + await matchSpanTreeSnapshot(selectedEvents(events), snapshotPath); + }); + }); +} diff --git a/e2e/scenarios/ollama-instrumentation/constants.mjs b/e2e/scenarios/ollama-instrumentation/constants.mjs new file mode 100644 index 000000000..025d1d866 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/constants.mjs @@ -0,0 +1,4 @@ +export const ROOT_NAME = "ollama-instrumentation-root"; +export const SCENARIO_NAME = "ollama-instrumentation"; +export const GENERATION_MODEL = "gpt-oss:20b"; +export const EMBEDDING_MODEL = "embeddinggemma"; diff --git a/e2e/scenarios/ollama-instrumentation/package.json b/e2e/scenarios/ollama-instrumentation/package.json new file mode 100644 index 000000000..e9a3b6934 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/package.json @@ -0,0 +1,18 @@ +{ + "name": "@braintrust/e2e-ollama-instrumentation", + "private": true, + "braintrustScenario": { + "bump": { + "dependencies": { + "ollama-v0-6-latest": { + "package": "ollama", + "range": "0.6" + } + } + } + }, + "dependencies": { + "ollama-v0-6": "npm:ollama@0.6.3", + "ollama-v0-6-latest": "npm:ollama@0.6.3" + } +} diff --git a/e2e/scenarios/ollama-instrumentation/pnpm-lock.yaml b/e2e/scenarios/ollama-instrumentation/pnpm-lock.yaml new file mode 100644 index 000000000..d90313710 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/pnpm-lock.yaml @@ -0,0 +1,32 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + ollama-v0-6: + specifier: npm:ollama@0.6.3 + version: ollama@0.6.3 + ollama-v0-6-latest: + specifier: npm:ollama@0.6.3 + version: ollama@0.6.3 + +packages: + + ollama@0.6.3: + resolution: {integrity: sha512-KEWEhIqE5wtfzEIZbDCLH51VFZ6Z3ZSa6sIOg/E/tBV8S51flyqBOXi+bRxlOYKDf8i327zG9eSTb8IJxvm3Zg==} + + whatwg-fetch@3.6.20: + resolution: {integrity: sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==} + +snapshots: + + ollama@0.6.3: + dependencies: + whatwg-fetch: 3.6.20 + + whatwg-fetch@3.6.20: {} diff --git a/e2e/scenarios/ollama-instrumentation/scenario.impl.mjs b/e2e/scenarios/ollama-instrumentation/scenario.impl.mjs new file mode 100644 index 000000000..7595b4cdc --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/scenario.impl.mjs @@ -0,0 +1,238 @@ +import { wrapOllama } from "braintrust"; +import { + collectAsync, + runOperation, + runTracedScenario, +} from "../../helpers/provider-runtime.mjs"; +import { + EMBEDDING_MODEL, + GENERATION_MODEL, + ROOT_NAME, + SCENARIO_NAME, +} from "./constants.mjs"; + +export const OLLAMA_SCENARIO_TIMEOUT_MS = 240_000; + +function requireOllamaConfig() { + const apiKey = process.env.OLLAMA_API_KEY; + const host = process.env.OLLAMA_HOST; + if (!apiKey) { + throw new Error("Expected OLLAMA_API_KEY to be set for e2e"); + } + if (!host) { + throw new Error("Expected OLLAMA_HOST to be set for e2e"); + } + return { apiKey, host }; +} + +function jsonResponse(body, status = 200) { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); +} + +async function syntheticFetch(input, init) { + const url = String(input); + const body = + typeof init?.body === "string" && init.body.length > 0 + ? JSON.parse(init.body) + : {}; + + if (url.endsWith("/api/embed")) { + return jsonResponse({ + model: body.model, + embeddings: [ + [0.1, 0.2, 0.3, 0.4], + [0.5, 0.6, 0.7, 0.8], + ], + prompt_eval_count: 4, + total_duration: 100, + load_duration: 10, + }); + } + + if (url.endsWith("/api/chat") && Array.isArray(body.tools)) { + return jsonResponse({ + model: body.model, + message: { + role: "assistant", + content: "", + tool_calls: [ + { + function: { + name: "get_temperature", + arguments: { city: "Paris" }, + }, + }, + ], + }, + done: true, + done_reason: "stop", + prompt_eval_count: 12, + eval_count: 5, + }); + } + + return jsonResponse({ error: "synthetic Ollama failure" }, 500); +} + +function generationOptions() { + return { + temperature: 0, + num_predict: 128, + }; +} + +async function runOllamaInstrumentationScenario( + Ollama, + { decorateClient } = {}, +) { + const { apiKey, host } = requireOllamaConfig(); + const headers = { Authorization: `Bearer ${apiKey}` }; + const liveBaseClient = new Ollama({ host, headers }); + const syntheticBaseClient = new Ollama({ + host: "http://synthetic.ollama.test", + headers, + fetch: syntheticFetch, + }); + const liveClient = decorateClient + ? decorateClient(liveBaseClient) + : liveBaseClient; + const syntheticClient = decorateClient + ? decorateClient(syntheticBaseClient) + : syntheticBaseClient; + + await runTracedScenario({ + callback: async () => { + await runOperation("ollama-chat-operation", "chat", async () => { + const response = await liveClient.chat({ + model: GENERATION_MODEL, + messages: [{ role: "user", content: "Reply with exactly OK." }], + options: generationOptions(), + think: false, + }); + if (!response?.message?.content) { + throw new Error("Expected Ollama chat response content"); + } + }); + + await runOperation( + "ollama-chat-stream-operation", + "chat-stream", + async () => { + const stream = await liveClient.chat({ + model: GENERATION_MODEL, + messages: [{ role: "user", content: "Reply with exactly STREAM." }], + options: generationOptions(), + stream: true, + think: false, + }); + const chunks = await collectAsync(stream); + if (chunks.length === 0) { + throw new Error("Expected Ollama chat stream chunks"); + } + }, + ); + + await runOperation("ollama-generate-operation", "generate", async () => { + const response = await liveClient.generate({ + model: GENERATION_MODEL, + prompt: "Reply with exactly OK.", + options: generationOptions(), + think: false, + }); + if (!response?.response) { + throw new Error("Expected Ollama generate response content"); + } + }); + + await runOperation( + "ollama-generate-stream-operation", + "generate-stream", + async () => { + const stream = await liveClient.generate({ + model: GENERATION_MODEL, + prompt: "Reply with exactly STREAM.", + options: generationOptions(), + stream: true, + think: false, + }); + const chunks = await collectAsync(stream); + if (chunks.length === 0) { + throw new Error("Expected Ollama generate stream chunks"); + } + }, + ); + + await runOperation( + "ollama-tool-call-operation", + "tool-call", + async () => { + await syntheticClient.chat({ + model: GENERATION_MODEL, + messages: [ + { + role: "user", + content: "Use get_temperature for Paris.", + }, + ], + tools: [ + { + type: "function", + function: { + name: "get_temperature", + description: "Get the temperature for a city", + parameters: { + type: "object", + properties: { + city: { type: "string" }, + }, + required: ["city"], + }, + }, + }, + ], + }); + }, + ); + + await runOperation("ollama-embed-operation", "embed", async () => { + await syntheticClient.embed({ + model: EMBEDDING_MODEL, + input: ["braintrust tracing", "ollama instrumentation"], + }); + }); + + await runOperation("ollama-error-operation", "error", async () => { + try { + await syntheticClient.chat({ + model: GENERATION_MODEL, + messages: [{ role: "user", content: "Trigger an error." }], + }); + throw new Error("Expected synthetic Ollama request to fail"); + } catch (error) { + if ( + error instanceof Error && + error.message === "Expected synthetic Ollama request to fail" + ) { + throw error; + } + } + }); + }, + metadata: { scenario: SCENARIO_NAME }, + projectNameBase: "e2e-ollama-instrumentation", + rootName: ROOT_NAME, + }); +} + +export async function runWrappedOllamaInstrumentation(Ollama) { + await runOllamaInstrumentationScenario(Ollama, { + decorateClient: wrapOllama, + }); +} + +export async function runAutoOllamaInstrumentation(Ollama) { + await runOllamaInstrumentationScenario(Ollama); +} diff --git a/e2e/scenarios/ollama-instrumentation/scenario.mjs b/e2e/scenarios/ollama-instrumentation/scenario.mjs new file mode 100644 index 000000000..4d3ef4ae0 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/scenario.mjs @@ -0,0 +1,10 @@ +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoOllamaInstrumentation } from "./scenario.impl.mjs"; + +const ollamaPackageName = + process.env.OLLAMA_PACKAGE_NAME ?? "ollama-v0-6-latest"; + +runMain(async () => { + const { Ollama } = await import(ollamaPackageName); + await runAutoOllamaInstrumentation(Ollama); +}); diff --git a/e2e/scenarios/ollama-instrumentation/scenario.test.ts b/e2e/scenarios/ollama-instrumentation/scenario.test.ts new file mode 100644 index 000000000..525d0f504 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/scenario.test.ts @@ -0,0 +1,76 @@ +import { describe } from "vitest"; +import { + prepareScenarioDir, + readInstalledPackageVersion, + resolveScenarioDir, +} from "../../helpers/scenario-harness"; +import { defineOllamaInstrumentationAssertions } from "./assertions"; +import { OLLAMA_SCENARIO_TIMEOUT_MS } from "./scenario.impl.mjs"; + +const originalScenarioDir = resolveScenarioDir(import.meta.url); +const scenarioDir = await prepareScenarioDir({ + scenarioDir: originalScenarioDir, +}); +const ollamaScenarios = await Promise.all( + [ + { + dependencyName: "ollama-v0-6", + variantKey: "ollama-v0.6", + }, + { + dependencyName: "ollama-v0-6-latest", + variantKey: "ollama-v0.6-latest", + }, + ].map(async (scenario) => ({ + ...scenario, + version: await readInstalledPackageVersion( + scenarioDir, + scenario.dependencyName, + ), + })), +); + +describe.sequential("variants", () => { + for (const scenario of ollamaScenarios) { + describe.sequential(`ollama sdk ${scenario.version}`, () => { + defineOllamaInstrumentationAssertions({ + name: "wrapped instrumentation", + runScenario: async ({ runScenarioDir }) => { + await runScenarioDir({ + entry: "scenario.ts", + env: { OLLAMA_PACKAGE_NAME: scenario.dependencyName }, + runContext: { + variantKey: scenario.variantKey, + originalScenarioDir, + }, + scenarioDir, + timeoutMs: OLLAMA_SCENARIO_TIMEOUT_MS, + }); + }, + snapshotName: `${scenario.variantKey}-wrapped`, + testFileUrl: import.meta.url, + timeoutMs: OLLAMA_SCENARIO_TIMEOUT_MS, + }); + + defineOllamaInstrumentationAssertions({ + name: "auto-hook instrumentation", + runScenario: async ({ runNodeScenarioDir }) => { + await runNodeScenarioDir({ + entry: "scenario.mjs", + env: { OLLAMA_PACKAGE_NAME: scenario.dependencyName }, + nodeArgs: ["--import", "braintrust/hook.mjs"], + runContext: { + variantKey: scenario.variantKey, + originalScenarioDir, + }, + scenarioDir, + timeoutMs: OLLAMA_SCENARIO_TIMEOUT_MS, + }); + }, + snapshotName: `${scenario.variantKey}-auto`, + testFileUrl: import.meta.url, + timeoutMs: OLLAMA_SCENARIO_TIMEOUT_MS, + }); + }); + } +}); diff --git a/e2e/scenarios/ollama-instrumentation/scenario.ts b/e2e/scenarios/ollama-instrumentation/scenario.ts new file mode 100644 index 000000000..6fb1e33d4 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/scenario.ts @@ -0,0 +1,10 @@ +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedOllamaInstrumentation } from "./scenario.impl.mjs"; + +const ollamaPackageName = + process.env.OLLAMA_PACKAGE_NAME ?? "ollama-v0-6-latest"; + +runMain(async () => { + const { Ollama } = await import(ollamaPackageName); + await runWrappedOllamaInstrumentation(Ollama); +}); diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.json index d60a9e008..664950e5b 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.json @@ -17,7 +17,7 @@ "children": [], "input": [ { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\nSpan names should generally remain stable across calls and versions. Do not include dynamic values such as model names in span names; record those values in metadata instead.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", "role": "system" }, { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.txt index 3f1b6eba9..ed7efd569 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.txt @@ -38,7 +38,7 @@ span_tree: ├── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\nSpan names should generally remain stable across calls and versions. Do not include dynamic values such as model names in span names; record those values in metadata instead.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", │ "role": "system" │ }, │ { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json index 13dc4a954..7569fe54b 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json @@ -17,7 +17,7 @@ "children": [], "input": [ { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\nSpan names should generally remain stable across calls and versions. Do not include dynamic values such as model names in span names; record those values in metadata instead.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", "role": "system" }, { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt index 747491000..4db394c56 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt @@ -38,7 +38,7 @@ span_tree: ├── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\nSpan names should generally remain stable across calls and versions. Do not include dynamic values such as model names in span names; record those values in metadata instead.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", │ "role": "system" │ }, │ { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json index 13dc4a954..7569fe54b 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json @@ -17,7 +17,7 @@ "children": [], "input": [ { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\nSpan names should generally remain stable across calls and versions. Do not include dynamic values such as model names in span names; record those values in metadata instead.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", "role": "system" }, { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt index 747491000..4db394c56 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt @@ -38,7 +38,7 @@ span_tree: ├── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\nSpan names should generally remain stable across calls and versions. Do not include dynamic values such as model names in span names; record those values in metadata instead.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", │ "role": "system" │ }, │ { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.json index d60a9e008..664950e5b 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.json @@ -17,7 +17,7 @@ "children": [], "input": [ { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\nSpan names should generally remain stable across calls and versions. Do not include dynamic values such as model names in span names; record those values in metadata instead.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", "role": "system" }, { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.txt index 3f1b6eba9..ed7efd569 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.txt @@ -38,7 +38,7 @@ span_tree: ├── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\nSpan names should generally remain stable across calls and versions. Do not include dynamic values such as model names in span names; record those values in metadata instead.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", │ "role": "system" │ }, │ { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.json deleted file mode 100644 index ab2c68bb1..000000000 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "span_tree": [ - { - "name": "pi-coding-agent-root", - "type": "task", - "children": [ - { - "name": "pi-coding-agent-prompt-operation", - "children": [ - { - "name": "AgentSession.prompt", - "type": "task", - "children": [ - { - "name": "anthropic.messages.create", - "type": "llm", - "children": [], - "input": [ - { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", - "role": "system" - }, - { - "content": [ - { - "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", - "type": "text" - } - ], - "role": "user" - }, - { - "content": null, - "role": "assistant", - "tool_calls": [ - { - "function": { - "arguments": "{\"command\":\"printf pi_tool_ok\"}", - "name": "bash" - }, - "id": "", - "type": "function" - } - ] - }, - { - "content": [ - { - "text": "pi_tool_ok", - "type": "text" - } - ], - "role": "tool", - "tool_call_id": "" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", - "role": "assistant" - } - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "pi_coding_agent.api": "anthropic-messages", - "pi_coding_agent.model": "claude-haiku-4-5", - "pi_coding_agent.operation": "agent.streamFn", - "pi_coding_agent.stop_reason": "stop", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 23, - "duration": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 2299, - "time_to_first_token": 0, - "tokens": 2322 - } - }, - { - "name": "bash", - "type": "tool", - "children": [], - "input": { - "command": "printf pi_tool_ok" - }, - "output": { - "content": [ - { - "text": "pi_tool_ok", - "type": "text" - } - ] - }, - "metadata": { - "gen_ai.tool.call.id": "", - "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" - } - } - ], - "input": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", - "role": "assistant" - } - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "pi_coding_agent.api": "anthropic-messages", - "pi_coding_agent.model": "claude-haiku-4-5", - "pi_coding_agent.operation": "AgentSession.prompt", - "pi_coding_agent.source": "rpc", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 80, - "duration": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 4524, - "tokens": 4604 - } - } - ], - "metadata": { - "operation": "prompt" - } - } - ], - "metadata": { - "scenario": "pi-coding-agent-instrumentation" - } - } - ] -} diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.txt deleted file mode 100644 index 1b3c2b6e4..000000000 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.txt +++ /dev/null @@ -1,121 +0,0 @@ -span_tree: -└── pi-coding-agent-root [task] - metadata: { - "scenario": "pi-coding-agent-instrumentation" - } - └── pi-coding-agent-prompt-operation - metadata: { - "operation": "prompt" - } - └── AgentSession.prompt [task] - input: "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output." - output: [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", - "role": "assistant" - } - } - ] - metadata: { - "model": "claude-haiku-4-5", - "pi_coding_agent.api": "anthropic-messages", - "pi_coding_agent.model": "claude-haiku-4-5", - "pi_coding_agent.operation": "AgentSession.prompt", - "pi_coding_agent.source": "rpc", - "provider": "anthropic" - } - metrics: { - "completion_tokens": 80, - "duration": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 4524, - "tokens": 4604 - } - ├── anthropic.messages.create [llm] - │ input: [ - │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", - │ "role": "system" - │ }, - │ { - │ "content": [ - │ { - │ "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ }, - │ { - │ "content": null, - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"command\":\"printf pi_tool_ok\"}", - │ "name": "bash" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ }, - │ { - │ "content": [ - │ { - │ "text": "pi_tool_ok", - │ "type": "text" - │ } - │ ], - │ "role": "tool", - │ "tool_call_id": "" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "message": { - │ "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "model": "claude-haiku-4-5", - │ "pi_coding_agent.api": "anthropic-messages", - │ "pi_coding_agent.model": "claude-haiku-4-5", - │ "pi_coding_agent.operation": "agent.streamFn", - │ "pi_coding_agent.stop_reason": "stop", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 23, - │ "duration": 0, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 2299, - │ "time_to_first_token": 0, - │ "tokens": 2322 - │ } - └── bash [tool] - input: { - "command": "printf pi_tool_ok" - } - output: { - "content": [ - { - "text": "pi_tool_ok", - "type": "text" - } - ] - } - metadata: { - "gen_ai.tool.call.id": "", - "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" - } diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.json deleted file mode 100644 index ab2c68bb1..000000000 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "span_tree": [ - { - "name": "pi-coding-agent-root", - "type": "task", - "children": [ - { - "name": "pi-coding-agent-prompt-operation", - "children": [ - { - "name": "AgentSession.prompt", - "type": "task", - "children": [ - { - "name": "anthropic.messages.create", - "type": "llm", - "children": [], - "input": [ - { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", - "role": "system" - }, - { - "content": [ - { - "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", - "type": "text" - } - ], - "role": "user" - }, - { - "content": null, - "role": "assistant", - "tool_calls": [ - { - "function": { - "arguments": "{\"command\":\"printf pi_tool_ok\"}", - "name": "bash" - }, - "id": "", - "type": "function" - } - ] - }, - { - "content": [ - { - "text": "pi_tool_ok", - "type": "text" - } - ], - "role": "tool", - "tool_call_id": "" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", - "role": "assistant" - } - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "pi_coding_agent.api": "anthropic-messages", - "pi_coding_agent.model": "claude-haiku-4-5", - "pi_coding_agent.operation": "agent.streamFn", - "pi_coding_agent.stop_reason": "stop", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 23, - "duration": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 2299, - "time_to_first_token": 0, - "tokens": 2322 - } - }, - { - "name": "bash", - "type": "tool", - "children": [], - "input": { - "command": "printf pi_tool_ok" - }, - "output": { - "content": [ - { - "text": "pi_tool_ok", - "type": "text" - } - ] - }, - "metadata": { - "gen_ai.tool.call.id": "", - "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" - } - } - ], - "input": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", - "role": "assistant" - } - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "pi_coding_agent.api": "anthropic-messages", - "pi_coding_agent.model": "claude-haiku-4-5", - "pi_coding_agent.operation": "AgentSession.prompt", - "pi_coding_agent.source": "rpc", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 80, - "duration": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 4524, - "tokens": 4604 - } - } - ], - "metadata": { - "operation": "prompt" - } - } - ], - "metadata": { - "scenario": "pi-coding-agent-instrumentation" - } - } - ] -} diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.txt deleted file mode 100644 index 1b3c2b6e4..000000000 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.txt +++ /dev/null @@ -1,121 +0,0 @@ -span_tree: -└── pi-coding-agent-root [task] - metadata: { - "scenario": "pi-coding-agent-instrumentation" - } - └── pi-coding-agent-prompt-operation - metadata: { - "operation": "prompt" - } - └── AgentSession.prompt [task] - input: "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output." - output: [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", - "role": "assistant" - } - } - ] - metadata: { - "model": "claude-haiku-4-5", - "pi_coding_agent.api": "anthropic-messages", - "pi_coding_agent.model": "claude-haiku-4-5", - "pi_coding_agent.operation": "AgentSession.prompt", - "pi_coding_agent.source": "rpc", - "provider": "anthropic" - } - metrics: { - "completion_tokens": 80, - "duration": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 4524, - "tokens": 4604 - } - ├── anthropic.messages.create [llm] - │ input: [ - │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", - │ "role": "system" - │ }, - │ { - │ "content": [ - │ { - │ "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ }, - │ { - │ "content": null, - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"command\":\"printf pi_tool_ok\"}", - │ "name": "bash" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ }, - │ { - │ "content": [ - │ { - │ "text": "pi_tool_ok", - │ "type": "text" - │ } - │ ], - │ "role": "tool", - │ "tool_call_id": "" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "message": { - │ "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "model": "claude-haiku-4-5", - │ "pi_coding_agent.api": "anthropic-messages", - │ "pi_coding_agent.model": "claude-haiku-4-5", - │ "pi_coding_agent.operation": "agent.streamFn", - │ "pi_coding_agent.stop_reason": "stop", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 23, - │ "duration": 0, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 2299, - │ "time_to_first_token": 0, - │ "tokens": 2322 - │ } - └── bash [tool] - input: { - "command": "printf pi_tool_ok" - } - output: { - "content": [ - { - "text": "pi_tool_ok", - "type": "text" - } - ] - } - metadata: { - "gen_ai.tool.call.id": "", - "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" - } diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/scenario.impl.mjs b/e2e/scenarios/pi-coding-agent-instrumentation/scenario.impl.mjs index aa6f28014..80a35d9c0 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/pi-coding-agent-instrumentation/scenario.impl.mjs @@ -9,8 +9,13 @@ export const SCENARIO_NAME = "pi-coding-agent-instrumentation"; async function runPiCodingAgentScenario({ decorateSDK, sdk }) { const instrumentedSDK = decorateSDK ? decorateSDK(sdk) : sdk; - const { AuthStorage, ModelRegistry, SessionManager, createAgentSession } = - instrumentedSDK; + const { + AuthStorage, + DefaultResourceLoader, + ModelRegistry, + SessionManager, + createAgentSession, + } = instrumentedSDK; const authStorage = AuthStorage.inMemory(); authStorage.setRuntimeApiKey("anthropic", process.env.ANTHROPIC_API_KEY); @@ -22,6 +27,13 @@ async function runPiCodingAgentScenario({ decorateSDK, sdk }) { if (!model) { throw new Error("Expected Pi Coding Agent Anthropic model"); } + const cwd = process.cwd(); + const resourceLoader = new DefaultResourceLoader({ + agentDir: cwd, + cwd, + noContextFiles: true, + }); + await resourceLoader.reload(); let session; await runTracedScenario({ @@ -32,10 +44,11 @@ async function runPiCodingAgentScenario({ decorateSDK, sdk }) { async () => { const result = await createAgentSession({ authStorage, - cwd: process.cwd(), + cwd, model, modelRegistry, - sessionManager: SessionManager.inMemory(process.cwd()), + resourceLoader, + sessionManager: SessionManager.inMemory(cwd), thinkingLevel: "off", tools: ["bash"], }); diff --git a/js/src/auto-instrumentations/configs/all.ts b/js/src/auto-instrumentations/configs/all.ts index 9944ebab3..42692d971 100644 --- a/js/src/auto-instrumentations/configs/all.ts +++ b/js/src/auto-instrumentations/configs/all.ts @@ -24,6 +24,7 @@ import { huggingFaceTransformersConfigs } from "./huggingface-transformers"; import { langchainConfigs } from "./langchain"; import { langSmithConfigs } from "./langsmith"; import { mistralConfigs } from "./mistral"; +import { ollamaConfigs } from "./ollama"; import { openAIAgentsCoreConfigs } from "./openai-agents"; import { openaiConfigs } from "./openai"; import { openAICodexConfigs } from "./openai-codex"; @@ -93,6 +94,7 @@ const defaultInstrumentationConfigGroups: readonly InstrumentationConfigGroup[] configs: openRouterAgentConfigs, }, { integrations: ["mistral"], configs: mistralConfigs }, + { integrations: ["ollama"], configs: ollamaConfigs }, { integrations: ["googleADK"], configs: googleADKConfigs }, { integrations: ["cohere"], configs: cohereConfigs }, { integrations: ["groq"], configs: groqConfigs }, diff --git a/js/src/auto-instrumentations/configs/ollama.ts b/js/src/auto-instrumentations/configs/ollama.ts new file mode 100644 index 000000000..5ef28eb64 --- /dev/null +++ b/js/src/auto-instrumentations/configs/ollama.ts @@ -0,0 +1,31 @@ +import type { InstrumentationConfig } from "../orchestrion-js"; +import { ollamaChannels } from "../../instrumentation/plugins/ollama-channels"; + +const methods = [ + ["chat", ollamaChannels.chat.channelName], + ["generate", ollamaChannels.generate.channelName], + ["embed", ollamaChannels.embed.channelName], +] as const; + +// Ollama's Node entry points in dist/index.{mjs,cjs} subclass the Ollama class +// exported here. These methods are defined on that shared base class, so +// instrumenting these files covers both the Node and browser package exports. +const implementationFiles = ["dist/browser.mjs", "dist/browser.cjs"] as const; + +export const ollamaConfigs: InstrumentationConfig[] = [ + ...implementationFiles.flatMap((filePath) => + methods.map(([methodName, channelName]) => ({ + channelName, + module: { + name: "ollama", + versionRange: ">=0.6.0 <0.7.0", + filePath, + }, + functionQuery: { + className: "Ollama", + methodName, + kind: "Async" as const, + }, + })), + ), +]; diff --git a/js/src/auto-instrumentations/index.ts b/js/src/auto-instrumentations/index.ts index 2e1f68d6a..3a59b73e4 100644 --- a/js/src/auto-instrumentations/index.ts +++ b/js/src/auto-instrumentations/index.ts @@ -43,6 +43,7 @@ export { huggingFaceConfigs } from "./configs/huggingface"; export { openRouterAgentConfigs } from "./configs/openrouter-agent"; export { openRouterConfigs } from "./configs/openrouter"; export { mistralConfigs } from "./configs/mistral"; +export { ollamaConfigs } from "./configs/ollama"; export { googleADKConfigs } from "./configs/google-adk"; export { cloudflareAIChatConfigs } from "./configs/cloudflare-ai-chat"; export { cohereConfigs } from "./configs/cohere"; diff --git a/js/src/exports.ts b/js/src/exports.ts index abd2bf611..e8bad926e 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -222,6 +222,7 @@ export { wrapHuggingFaceTransformers } from "./wrappers/huggingface-transformers export { wrapOpenRouterAgent } from "./wrappers/openrouter-agent"; export { wrapOpenRouter } from "./wrappers/openrouter"; export { wrapMistral } from "./wrappers/mistral"; +export { wrapOllama } from "./wrappers/ollama"; export { wrapCohere } from "./wrappers/cohere"; export { wrapGroq } from "./wrappers/groq"; export { wrapBedrockRuntime } from "./wrappers/bedrock-runtime"; diff --git a/js/src/instrumentation/braintrust-plugin.test.ts b/js/src/instrumentation/braintrust-plugin.test.ts index f70511492..30657391d 100644 --- a/js/src/instrumentation/braintrust-plugin.test.ts +++ b/js/src/instrumentation/braintrust-plugin.test.ts @@ -13,6 +13,7 @@ import { HuggingFaceTransformersPlugin } from "./plugins/huggingface-transformer import { OpenRouterAgentPlugin } from "./plugins/openrouter-agent-plugin"; import { OpenRouterPlugin } from "./plugins/openrouter-plugin"; import { MistralPlugin } from "./plugins/mistral-plugin"; +import { OllamaPlugin } from "./plugins/ollama-plugin"; import { CoherePlugin } from "./plugins/cohere-plugin"; import { GroqPlugin } from "./plugins/groq-plugin"; import { GitHubCopilotPlugin } from "./plugins/github-copilot-plugin"; @@ -95,6 +96,10 @@ vi.mock("./plugins/mistral-plugin", () => ({ MistralPlugin: createPluginClassMock(), })); +vi.mock("./plugins/ollama-plugin", () => ({ + OllamaPlugin: createPluginClassMock(), +})); + vi.mock("./plugins/cohere-plugin", () => ({ CoherePlugin: createPluginClassMock(), })); @@ -568,6 +573,18 @@ describe("BraintrustPlugin", () => { expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); }); + it("should not create Ollama plugin when ollama: false", () => { + const plugin = new BraintrustPlugin({ + integrations: { ollama: false }, + }); + plugin.enable(); + + expect(OllamaPlugin).not.toHaveBeenCalled(); + expect(OpenAIPlugin).toHaveBeenCalledTimes(1); + expect(AnthropicPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); + }); + it("should not create Cohere plugin when cohere: false", () => { const plugin = new BraintrustPlugin({ integrations: { cohere: false }, @@ -674,6 +691,7 @@ describe("BraintrustPlugin", () => { openrouter: false, openrouterAgent: false, mistral: false, + ollama: false, cohere: false, groq: false, gitHubCopilot: false, @@ -699,6 +717,7 @@ describe("BraintrustPlugin", () => { expect(OpenRouterPlugin).not.toHaveBeenCalled(); expect(OpenRouterAgentPlugin).not.toHaveBeenCalled(); expect(MistralPlugin).not.toHaveBeenCalled(); + expect(OllamaPlugin).not.toHaveBeenCalled(); expect(CoherePlugin).not.toHaveBeenCalled(); expect(GroqPlugin).not.toHaveBeenCalled(); expect(GitHubCopilotPlugin).not.toHaveBeenCalled(); @@ -908,6 +927,7 @@ describe("BraintrustPlugin", () => { const openRouterAgentMock = vi.mocked(OpenRouterAgentPlugin).mock .results[0].value; const mistralMock = vi.mocked(MistralPlugin).mock.results[0].value; + const ollamaMock = vi.mocked(OllamaPlugin).mock.results[0].value; const cohereMock = vi.mocked(CoherePlugin).mock.results[0].value; const groqMock = vi.mocked(GroqPlugin).mock.results[0].value; const piCodingAgentMock = @@ -930,6 +950,7 @@ describe("BraintrustPlugin", () => { expect(openRouterMock.enable).toHaveBeenCalledTimes(1); expect(openRouterAgentMock.enable).toHaveBeenCalledTimes(1); expect(mistralMock.enable).toHaveBeenCalledTimes(1); + expect(ollamaMock.enable).toHaveBeenCalledTimes(1); expect(cohereMock.enable).toHaveBeenCalledTimes(1); expect(groqMock.enable).toHaveBeenCalledTimes(1); expect(piCodingAgentMock.enable).toHaveBeenCalledTimes(1); @@ -962,6 +983,7 @@ describe("BraintrustPlugin", () => { const openRouterAgentMock = vi.mocked(OpenRouterAgentPlugin).mock .results[0].value; const mistralMock = vi.mocked(MistralPlugin).mock.results[0].value; + const ollamaMock = vi.mocked(OllamaPlugin).mock.results[0].value; const cohereMock = vi.mocked(CoherePlugin).mock.results[0].value; const groqMock = vi.mocked(GroqPlugin).mock.results[0].value; const piCodingAgentMock = @@ -986,6 +1008,7 @@ describe("BraintrustPlugin", () => { expect(openRouterMock.disable).toHaveBeenCalledTimes(1); expect(openRouterAgentMock.disable).toHaveBeenCalledTimes(1); expect(mistralMock.disable).toHaveBeenCalledTimes(1); + expect(ollamaMock.disable).toHaveBeenCalledTimes(1); expect(cohereMock.disable).toHaveBeenCalledTimes(1); expect(groqMock.disable).toHaveBeenCalledTimes(1); expect(piCodingAgentMock.disable).toHaveBeenCalledTimes(1); @@ -1036,6 +1059,7 @@ describe("BraintrustPlugin", () => { expect(OpenRouterPlugin).not.toHaveBeenCalled(); expect(OpenRouterAgentPlugin).not.toHaveBeenCalled(); expect(MistralPlugin).not.toHaveBeenCalled(); + expect(OllamaPlugin).not.toHaveBeenCalled(); expect(CoherePlugin).not.toHaveBeenCalled(); expect(GroqPlugin).not.toHaveBeenCalled(); expect(PiCodingAgentPlugin).not.toHaveBeenCalled(); @@ -1062,6 +1086,7 @@ describe("BraintrustPlugin", () => { expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); expect(MistralPlugin).toHaveBeenCalledTimes(1); + expect(OllamaPlugin).toHaveBeenCalledTimes(1); expect(CoherePlugin).toHaveBeenCalledTimes(1); expect(GroqPlugin).toHaveBeenCalledTimes(1); expect(PiCodingAgentPlugin).toHaveBeenCalledTimes(1); @@ -1083,6 +1108,7 @@ describe("BraintrustPlugin", () => { openrouter: true, openrouterAgent: true, mistral: false, + ollama: false, cohere: false, groq: true, langchain: true, @@ -1116,6 +1142,7 @@ describe("BraintrustPlugin", () => { expect(groqMock.disable).toHaveBeenCalledTimes(1); expect(langChainMock.disable).toHaveBeenCalledTimes(1); expect(MistralPlugin).not.toHaveBeenCalled(); + expect(OllamaPlugin).not.toHaveBeenCalled(); expect(CoherePlugin).not.toHaveBeenCalled(); }); }); diff --git a/js/src/instrumentation/braintrust-plugin.ts b/js/src/instrumentation/braintrust-plugin.ts index 69575b2cc..6bf7af0b6 100644 --- a/js/src/instrumentation/braintrust-plugin.ts +++ b/js/src/instrumentation/braintrust-plugin.ts @@ -13,6 +13,7 @@ import { HuggingFaceTransformersPlugin } from "./plugins/huggingface-transformer import { OpenRouterAgentPlugin } from "./plugins/openrouter-agent-plugin"; import { OpenRouterPlugin } from "./plugins/openrouter-plugin"; import { MistralPlugin } from "./plugins/mistral-plugin"; +import { OllamaPlugin } from "./plugins/ollama-plugin"; import { GoogleADKPlugin } from "./plugins/google-adk-plugin"; import { CoherePlugin } from "./plugins/cohere-plugin"; import { GroqPlugin } from "./plugins/groq-plugin"; @@ -44,6 +45,7 @@ export interface BraintrustPluginConfig { * - HuggingFace Inference SDK * - LangChain.js and LangGraph * - Mistral SDK + * - Ollama SDK * - Cohere SDK * * The plugin is automatically enabled when the Braintrust library is loaded. @@ -66,6 +68,7 @@ export class BraintrustPlugin extends BasePlugin { private openRouterPlugin: OpenRouterPlugin | null = null; private openRouterAgentPlugin: OpenRouterAgentPlugin | null = null; private mistralPlugin: MistralPlugin | null = null; + private ollamaPlugin: OllamaPlugin | null = null; private googleADKPlugin: GoogleADKPlugin | null = null; private coherePlugin: CoherePlugin | null = null; private groqPlugin: GroqPlugin | null = null; @@ -163,6 +166,11 @@ export class BraintrustPlugin extends BasePlugin { this.mistralPlugin.enable(); } + if (integrations.ollama !== false) { + this.ollamaPlugin = new OllamaPlugin(); + this.ollamaPlugin.enable(); + } + // Enable Google ADK integration (default: true) if (integrations.googleADK !== false) { this.googleADKPlugin = new GoogleADKPlugin(); @@ -313,6 +321,11 @@ export class BraintrustPlugin extends BasePlugin { this.mistralPlugin = null; } + if (this.ollamaPlugin) { + this.ollamaPlugin.disable(); + this.ollamaPlugin = null; + } + if (this.googleADKPlugin) { this.googleADKPlugin.disable(); this.googleADKPlugin = null; diff --git a/js/src/instrumentation/config.ts b/js/src/instrumentation/config.ts index 15f454578..77875ac2a 100644 --- a/js/src/instrumentation/config.ts +++ b/js/src/instrumentation/config.ts @@ -18,6 +18,7 @@ export interface InstrumentationIntegrationsConfig { openrouter?: boolean; openrouterAgent?: boolean; mistral?: boolean; + ollama?: boolean; cohere?: boolean; groq?: boolean; bedrock?: boolean; @@ -95,6 +96,7 @@ const envIntegrationAliases: Record< openrouteragent: "openrouterAgent", "openrouter-agent": "openrouterAgent", mistral: "mistral", + ollama: "ollama", googleadk: "googleADK", "google-adk": "googleADK", cohere: "cohere", @@ -143,6 +145,7 @@ export function getDefaultInstrumentationIntegrations(): Record< openrouter: true, openrouterAgent: true, mistral: true, + ollama: true, cohere: true, groq: true, bedrock: true, diff --git a/js/src/instrumentation/core/channel-tracing.test.ts b/js/src/instrumentation/core/channel-tracing.test.ts index 5af10b120..55c08f05b 100644 --- a/js/src/instrumentation/core/channel-tracing.test.ts +++ b/js/src/instrumentation/core/channel-tracing.test.ts @@ -314,4 +314,48 @@ describe("traceAsyncChannel current span binding", () => { expect(onError).toHaveBeenCalledTimes(1); expect(end).toHaveBeenCalledTimes(2); }); + + it("records stream cancellation as an error", async () => { + const onError = vi.fn(); + const child = { + end: vi.fn(), + log: vi.fn(), + } as unknown as Span; + const unsubscribe = traceStreamingChannel(testChannels.streamingCall, { + name: "streaming-channel-test", + startSpan: () => child, + type: "function", + extractInput: () => ({ input: "input", metadata: undefined }), + extractOutput: (result) => result, + extractMetrics: () => ({}), + onError, + }); + const stream = { + abort: vi.fn(), + async *[Symbol.asyncIterator]() { + yield { ok: true }; + }, + }; + + try { + const patched = await testChannels.streamingCall.tracePromise( + async () => stream as any, + { arguments: [{}] } as any, + ); + (patched as unknown as typeof stream).abort(); + await Promise.resolve(); + } finally { + unsubscribe(); + } + + const cancellationError = expect.objectContaining({ + message: "Stream cancelled before completion", + name: "AbortError", + }); + expect(child.log).toHaveBeenLastCalledWith({ error: cancellationError }); + expect(child.end).toHaveBeenCalledOnce(); + expect(onError).toHaveBeenCalledWith( + expect.objectContaining({ error: cancellationError }), + ); + }); }); diff --git a/js/src/instrumentation/core/channel-tracing.ts b/js/src/instrumentation/core/channel-tracing.ts index 8b200d651..a8ecc819c 100644 --- a/js/src/instrumentation/core/channel-tracing.ts +++ b/js/src/instrumentation/core/channel-tracing.ts @@ -563,6 +563,33 @@ export function traceStreamingChannel( if (isAsyncIterable(asyncEndEvent.result)) { let firstChunkTime: number | undefined; + const handleStreamError = (error: Error) => { + try { + span.log({ error }); + } catch (loggingError) { + debugLogger.error( + `Error logging failure for ${channelName}:`, + loggingError, + ); + } + try { + span.end(); + } catch (endingError) { + debugLogger.error( + `Error ending span for ${channelName}:`, + endingError, + ); + } + states.delete(event as object); + runStreamingErrorHook({ + channelName, + config, + error, + event: asyncEndEvent, + span, + startTime, + }); + }; patchStreamIfNeeded(asyncEndEvent.result, { onChunk: () => { @@ -661,33 +688,12 @@ export function traceStreamingChannel( }); } }, - onError: (error: Error) => { - try { - span.log({ error }); - } catch (loggingError) { - debugLogger.error( - `Error logging failure for ${channelName}:`, - loggingError, - ); - } - try { - span.end(); - } catch (endingError) { - debugLogger.error( - `Error ending span for ${channelName}:`, - endingError, - ); - } - states.delete(event as object); - runStreamingErrorHook({ - channelName, - config, - error, - event: asyncEndEvent, - span, - startTime, - }); + onCancel: () => { + const error = new Error("Stream cancelled before completion"); + error.name = "AbortError"; + handleStreamError(error); }, + onError: handleStreamError, }); return; } diff --git a/js/src/instrumentation/core/stream-patcher.test.ts b/js/src/instrumentation/core/stream-patcher.test.ts index 8a71b90e5..f3954a41b 100644 --- a/js/src/instrumentation/core/stream-patcher.test.ts +++ b/js/src/instrumentation/core/stream-patcher.test.ts @@ -258,6 +258,78 @@ describe("patchStreamIfNeeded", () => { expect(onComplete).toHaveBeenCalledWith([1, 2]); }); + it("should cancel an abortable stream when abort() is called", async () => { + const iterator = { + values: [1, 2, 3], + index: 0, + async next() { + const value = this.values[this.index++]; + return value === undefined + ? { done: true as const, value: undefined } + : { done: false as const, value }; + }, + }; + const abort = vi.fn(() => "aborted"); + const stream = { + abort, + [Symbol.asyncIterator]() { + return iterator; + }, + }; + const onComplete = vi.fn(); + const onCancel = vi.fn(); + const patched = patchStreamIfNeeded(stream, { + onCancel, + onComplete, + }) as typeof stream; + const patchedIterator = patched[Symbol.asyncIterator](); + + await patchedIterator.next(); + await patchedIterator.next(); + + expect(patched.abort()).toBe("aborted"); + expect(abort).toHaveBeenCalledOnce(); + expect(onCancel).toHaveBeenCalledOnce(); + expect(onCancel).toHaveBeenCalledWith([1, 2]); + expect(onComplete).not.toHaveBeenCalled(); + + patched.abort(); + expect(onCancel).toHaveBeenCalledOnce(); + }); + + it("should complete an abortable self-iterator when abort() is called", async () => { + const abort = vi.fn(() => "aborted"); + const stream = { + values: [1, 2, 3], + index: 0, + abort, + async next() { + const value = this.values[this.index++]; + return value === undefined + ? { done: true as const, value: undefined } + : { done: false as const, value }; + }, + [Symbol.asyncIterator]() { + return this; + }, + }; + const onComplete = vi.fn(); + const patched = patchStreamIfNeeded(stream, { + onComplete, + }) as typeof stream; + + await patched.next(); + await patched.next(); + + expect(patched.abort()).toBe("aborted"); + expect(abort).toHaveBeenCalledOnce(); + expect(onComplete).toHaveBeenCalledOnce(); + expect(onComplete).toHaveBeenCalledWith([1, 2]); + + patched.abort(); + expect(onComplete).toHaveBeenCalledOnce(); + }); + it("should handle error injection via throw()", async () => { const stream = { async *[Symbol.asyncIterator]() { diff --git a/js/src/instrumentation/core/stream-patcher.ts b/js/src/instrumentation/core/stream-patcher.ts index 13807d5ca..8a92b509e 100644 --- a/js/src/instrumentation/core/stream-patcher.ts +++ b/js/src/instrumentation/core/stream-patcher.ts @@ -6,6 +6,8 @@ * even though they cannot replace return values. */ +import { debugLogger } from "../../debug-logger"; + /** * Check if a value is an async iterable (stream). */ @@ -36,6 +38,12 @@ interface StreamPatchOptions { */ onComplete: (chunks: TChunk[]) => TFinal | void | Promise; + /** + * Called when the consumer cancels the stream before it completes. + * Falls back to onComplete when omitted. + */ + onCancel?: (chunks: TChunk[]) => void | Promise; + /** * Called if the stream errors. * If not provided, errors are re-thrown after collection stops. @@ -142,6 +150,42 @@ export function patchStreamIfNeeded( return stream; } + const chunks: TChunk[] = []; + let completed = false; + const notifyCancellation = async () => { + try { + await (options.onCancel ?? options.onComplete)(chunks); + } catch (error) { + debugLogger.error("Error in stream cancellation handler:", error); + } + }; + const patchAbortIfPresent = () => { + try { + if ( + "abort" in stream && + typeof (stream as { abort?: unknown }).abort === "function" + ) { + const originalAbort = ( + stream as { abort: (...args: unknown[]) => unknown } + ).abort.bind(stream); + (stream as { abort: (...args: unknown[]) => unknown }).abort = ( + ...args + ) => { + try { + return originalAbort(...args); + } finally { + if (!completed) { + completed = true; + void notifyCancellation(); + } + } + }; + } + } catch (error) { + debugLogger.warn("Failed to patch stream abort method:", error); + } + }; + // Only patch iterator methods directly when the stream is its own iterator. // Some SDKs expose a separate iterator from Symbol.asyncIterator(); patching // stream.next in those cases is a no-op because consumers never call it. @@ -156,8 +200,6 @@ export function patchStreamIfNeeded( typeof stream.return === "function" ? stream.return.bind(stream) : null; const originalThrow = typeof stream.throw === "function" ? stream.throw.bind(stream) : null; - const chunks: TChunk[] = []; - let completed = false; stream.next = async (...args: [] | [undefined]) => { try { @@ -219,12 +261,7 @@ export function patchStreamIfNeeded( stream.return = async (...args: [unknown?]) => { if (!completed) { completed = true; - try { - await options.onComplete(chunks); - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error("Error in stream onComplete handler:", error); - } + await notifyCancellation(); } return originalReturn(...args); }; @@ -256,6 +293,7 @@ export function patchStreamIfNeeded( value: true, }); + patchAbortIfPresent(); return stream; } catch (error) { // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. @@ -278,8 +316,6 @@ export function patchStreamIfNeeded( const patchedIteratorFn = function (this: any) { const iterator = originalIteratorFn.call(this); const originalNext = iterator.next.bind(iterator); - const chunks: TChunk[] = []; - let completed = false; // Patch the next() method iterator.next = async function (...args: [] | [undefined]) { @@ -350,13 +386,7 @@ export function patchStreamIfNeeded( iterator.return = async function (...args: any[]) { if (!completed) { completed = true; - // Stream was cancelled/returned early - try { - await options.onComplete(chunks); - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error("Error in stream onComplete handler:", error); - } + await notifyCancellation(); } return originalReturn(...args); }; @@ -397,6 +427,7 @@ export function patchStreamIfNeeded( // Replace the Symbol.asyncIterator method (stream as any)[Symbol.asyncIterator] = patchedIteratorFn; + patchAbortIfPresent(); return stream; } catch (error) { // If patching fails for any reason, log warning and return original diff --git a/js/src/instrumentation/plugins/instrumentation-names.test.ts b/js/src/instrumentation/plugins/instrumentation-names.test.ts index da54de4e8..d8f4418b8 100644 --- a/js/src/instrumentation/plugins/instrumentation-names.test.ts +++ b/js/src/instrumentation/plugins/instrumentation-names.test.ts @@ -24,6 +24,7 @@ import { huggingFaceChannels } from "./huggingface-channels"; import { langChainChannels } from "./langchain-channels"; import { langSmithChannels } from "./langsmith-channels"; import { mistralChannels } from "./mistral-channels"; +import { ollamaChannels } from "./ollama-channels"; import { openAIAgentsCoreChannels } from "./openai-agents-channels"; import { openAIChannels } from "./openai-channels"; import { openAICodexChannels } from "./openai-codex-channels"; @@ -66,6 +67,7 @@ describe("built-in instrumentation provenance names", () => { [langChainChannels.configure, INSTRUMENTATION_NAMES.LANGCHAIN], [langSmithChannels.createRun, INSTRUMENTATION_NAMES.LANGSMITH], [mistralChannels.chatComplete, INSTRUMENTATION_NAMES.MISTRAL], + [ollamaChannels.chat, INSTRUMENTATION_NAMES.OLLAMA], [ openAIAgentsCoreChannels.onTraceStart, INSTRUMENTATION_NAMES.OPENAI_AGENTS, diff --git a/js/src/instrumentation/plugins/ollama-channels.ts b/js/src/instrumentation/plugins/ollama-channels.ts new file mode 100644 index 000000000..c6e4f2044 --- /dev/null +++ b/js/src/instrumentation/plugins/ollama-channels.ts @@ -0,0 +1,41 @@ +import { INSTRUMENTATION_NAMES } from "../../span-origin"; +import type { + OllamaChatRequest, + OllamaChatResponse, + OllamaChatResult, + OllamaEmbedRequest, + OllamaEmbedResponse, + OllamaGenerateRequest, + OllamaGenerateResponse, + OllamaGenerateResult, +} from "../../vendor-sdk-types/ollama"; +import { channel, defineChannels } from "../core/channel-definitions"; + +export const ollamaChannels = defineChannels( + "ollama", + { + chat: channel< + [OllamaChatRequest], + OllamaChatResult, + Record, + OllamaChatResponse + >({ + channelName: "chat", + kind: "async", + }), + generate: channel< + [OllamaGenerateRequest], + OllamaGenerateResult, + Record, + OllamaGenerateResponse + >({ + channelName: "generate", + kind: "async", + }), + embed: channel<[OllamaEmbedRequest], OllamaEmbedResponse>({ + channelName: "embed", + kind: "async", + }), + }, + { instrumentationName: INSTRUMENTATION_NAMES.OLLAMA }, +); diff --git a/js/src/instrumentation/plugins/ollama-plugin.test.ts b/js/src/instrumentation/plugins/ollama-plugin.test.ts new file mode 100644 index 000000000..0006efea8 --- /dev/null +++ b/js/src/instrumentation/plugins/ollama-plugin.test.ts @@ -0,0 +1,507 @@ +import { describe, expect, it, vi } from "vitest"; +import iso from "../../isomorph"; +import { configureNode } from "../../node/config"; +import { + aggregateOllamaChatChunks, + aggregateOllamaGenerateChunks, + extractOllamaChatInput, + extractOllamaChatOutput, + extractOllamaEmbedOutput, + extractOllamaGenerateInput, + extractOllamaMetrics, +} from "./ollama-plugin"; + +configureNode(); + +describe("Ollama instrumentation extraction", () => { + it("normalizes chat inputs, tools, and supported request metadata", () => { + const result = extractOllamaChatInput([ + { + model: "gpt-oss:20b", + messages: [ + { role: "system", content: "Be concise." }, + { role: "user", content: "What is the weather?" }, + { + role: "assistant", + content: "", + tool_calls: [ + { + function: { + name: "get_weather", + arguments: { city: "Paris" }, + }, + }, + { + function: { + name: "get_forecast", + arguments: { city: "Paris" }, + }, + }, + ], + }, + { + role: "tool", + tool_name: "get_forecast", + content: '{"forecast":"sunny"}', + }, + { + role: "tool", + tool_name: "get_weather", + content: '{"temperature":18}', + }, + { + role: "assistant", + content: "", + tool_calls: [ + { + function: { + name: "get_weather", + arguments: { city: "Paris" }, + }, + }, + ], + }, + { + role: "tool", + tool_name: "get_weather", + content: '{"temperature":19}', + }, + ], + tools: [ + { + type: "function", + function: { + name: "get_weather", + description: "Get weather", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + }, + }, + }, + ], + format: "json", + options: { + temperature: 0, + top_p: 0.8, + num_predict: 32, + frequency_penalty: 0.1, + presence_penalty: 0.2, + stop: ["DONE"], + top_k: 10, + }, + stream: true, + }, + ]); + + expect(result.input).toEqual([ + { role: "system", content: "Be concise." }, + { role: "user", content: "What is the weather?" }, + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "ollama_call_get_weather_0", + type: "function", + function: { + name: "get_weather", + arguments: '{"city":"Paris"}', + }, + }, + { + id: "ollama_call_get_forecast_1", + type: "function", + function: { + name: "get_forecast", + arguments: '{"city":"Paris"}', + }, + }, + ], + }, + { + role: "tool", + tool_call_id: "ollama_call_get_forecast_1", + content: '{"forecast":"sunny"}', + }, + { + role: "tool", + tool_call_id: "ollama_call_get_weather_0", + content: '{"temperature":18}', + }, + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "ollama_call_get_weather_2", + type: "function", + function: { + name: "get_weather", + arguments: '{"city":"Paris"}', + }, + }, + ], + }, + { + role: "tool", + tool_call_id: "ollama_call_get_weather_2", + content: '{"temperature":19}', + }, + ]); + expect(result.metadata).toEqual({ + provider: "ollama", + model: "gpt-oss:20b", + temperature: 0, + top_p: 0.8, + max_tokens: 32, + frequency_penalty: 0.1, + presence_penalty: 0.2, + stop: ["DONE"], + response_format: "json", + tools: [ + { + type: "function", + function: { + name: "get_weather", + description: "Get weather", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + }, + }, + }, + ], + }); + }); + + it("converts multimodal chat images into attachments", () => { + const result = extractOllamaChatInput([ + { + model: "llava", + messages: [ + { + role: "user", + content: "Describe this image.", + images: ["data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB"], + }, + ], + }, + ]); + + const content = ( + result.input as Array<{ + content: Array<{ + type: string; + image_url?: { url?: { reference?: unknown } }; + }>; + }> + )[0].content; + expect(content[0]).toEqual({ + type: "text", + text: "Describe this image.", + }); + expect(content[1]).toMatchObject({ + type: "image_url", + image_url: { + url: { + reference: { + type: "braintrust_attachment", + content_type: "image/png", + }, + }, + }, + }); + }); + + it("converts local image paths into attachments", () => { + const statSync = vi + .spyOn(iso, "statSync") + .mockReturnValue({ isFile: () => true }); + const result = extractOllamaGenerateInput([ + { + model: "llava", + prompt: "Describe this image.", + images: ["/tmp/example.webp"], + }, + ]); + statSync.mockRestore(); + + const content = ( + result.input as Array<{ + content: Array<{ + type: string; + image_url?: { url?: { reference?: unknown } }; + }>; + }> + )[0].content; + expect(content[1]).toMatchObject({ + type: "image_url", + image_url: { + url: { + reference: { + type: "braintrust_attachment", + filename: "example.webp", + content_type: "image/webp", + }, + }, + }, + }); + }); + + it("preserves image-looking strings that are not readable local paths", () => { + const statSync = vi.spyOn(iso, "statSync").mockImplementation(() => { + throw new Error("missing"); + }); + const result = extractOllamaGenerateInput([ + { + model: "llava", + prompt: "Describe this image.", + images: ["missing.webp"], + }, + ]); + statSync.mockRestore(); + + expect(result.input).toEqual([ + { + role: "user", + content: "Describe this image.", + images: ["missing.webp"], + }, + ]); + }); + + it("normalizes chat output and tool calls as OpenAI choices", () => { + expect( + extractOllamaChatOutput({ + model: "gpt-oss:20b", + message: { + role: "assistant", + content: "", + thinking: "I should call the tool.", + tool_calls: [ + { + function: { + name: "get_weather", + arguments: { city: "Paris" }, + }, + }, + ], + }, + done: true, + done_reason: "stop", + }), + ).toEqual([ + { + index: 0, + finish_reason: "tool_calls", + message: { + role: "assistant", + content: null, + reasoning: "I should call the tool.", + tool_calls: [ + { + id: "ollama_call_get_weather_0", + type: "function", + function: { + name: "get_weather", + arguments: '{"city":"Paris"}', + }, + }, + ], + }, + }, + ]); + + expect( + extractOllamaChatOutput( + { + message: { + role: "assistant", + content: "", + tool_calls: [ + { + function: { + name: "get_weather", + arguments: { city: "Paris" }, + }, + }, + ], + }, + done: true, + }, + 2, + ), + ).toEqual([ + { + index: 0, + finish_reason: "tool_calls", + message: { + role: "assistant", + content: null, + tool_calls: [ + { + id: "ollama_call_get_weather_2", + type: "function", + function: { + name: "get_weather", + arguments: '{"city":"Paris"}', + }, + }, + ], + }, + }, + ]); + }); + + it("normalizes raw generation input as canonical messages", () => { + expect( + extractOllamaGenerateInput([ + { + model: "gpt-oss:20b", + system: "Be concise.", + prompt: "Say OK.", + suffix: "Done.", + options: { temperature: 0, num_predict: 8 }, + }, + ]), + ).toEqual({ + input: [ + { role: "system", content: "Be concise." }, + { role: "user", content: "Say OK.", suffix: "Done." }, + ], + metadata: { + provider: "ollama", + model: "gpt-oss:20b", + temperature: 0, + max_tokens: 8, + }, + }); + }); + + it("extracts token metrics and ignores invalid counts", () => { + expect( + extractOllamaMetrics({ + prompt_eval_count: 7, + eval_count: 3, + }), + ).toEqual({ + prompt_tokens: 7, + completion_tokens: 3, + tokens: 10, + }); + expect( + extractOllamaMetrics({ + prompt_eval_count: -1, + eval_count: Number.NaN, + }), + ).toEqual({}); + expect( + extractOllamaMetrics({ + prompt_eval_count: Number.NaN, + eval_count: 3, + }), + ).toEqual({ completion_tokens: 3, tokens: 3 }); + expect( + extractOllamaMetrics({ + prompt_eval_count: 7, + eval_count: -1, + }), + ).toEqual({ prompt_tokens: 7, tokens: 7 }); + }); + + it("aggregates chat and generation streams into final outputs", () => { + const chat = aggregateOllamaChatChunks( + [ + { + model: "gpt-oss:20b", + message: { + role: "assistant", + content: "Hello", + thinking: "Check", + }, + done: false, + }, + { + model: "gpt-oss:20b", + message: { role: "assistant", content: "!", thinking: " done" }, + done: true, + done_reason: "stop", + prompt_eval_count: 5, + eval_count: 2, + }, + ], + undefined, + undefined, + 1, + ); + expect(chat.output).toEqual([ + { + index: 0, + finish_reason: "stop", + message: { + role: "assistant", + content: "Hello!", + reasoning: "Check done", + }, + }, + ]); + expect(chat.metrics).toMatchObject({ + prompt_tokens: 5, + completion_tokens: 2, + tokens: 7, + }); + expect(chat.metadata).toEqual({ model: "gpt-oss:20b" }); + + const generation = aggregateOllamaGenerateChunks( + [ + { + model: "gpt-oss:20b", + response: "O", + thinking: "Check", + done: false, + }, + { + model: "gpt-oss:20b", + response: "K", + thinking: " done", + done: true, + done_reason: "stop", + prompt_eval_count: 4, + eval_count: 2, + }, + ], + undefined, + undefined, + 1, + ); + expect(generation.output).toEqual([ + { + index: 0, + finish_reason: "stop", + message: { + role: "assistant", + content: "OK", + reasoning: "Check done", + }, + }, + ]); + expect(generation.metrics).toMatchObject({ + prompt_tokens: 4, + completion_tokens: 2, + tokens: 6, + }); + }); + + it("summarizes embedding responses", () => { + expect( + extractOllamaEmbedOutput({ + model: "embeddinggemma", + embeddings: [ + [0.1, 0.2, 0.3], + [0.4, 0.5, 0.6], + ], + }), + ).toEqual({ embedding_length: 3 }); + }); +}); diff --git a/js/src/instrumentation/plugins/ollama-plugin.ts b/js/src/instrumentation/plugins/ollama-plugin.ts new file mode 100644 index 000000000..f48040348 --- /dev/null +++ b/js/src/instrumentation/plugins/ollama-plugin.ts @@ -0,0 +1,642 @@ +import { SpanTypeAttribute, isObject } from "../../../util/index"; +import iso from "../../isomorph"; +import { Attachment } from "../../logger"; +import { processInputAttachments } from "../../wrappers/attachment-utils"; +import type { + OllamaChatRequest, + OllamaChatResponse, + OllamaEmbedRequest, + OllamaEmbedResponse, + OllamaGenerateRequest, + OllamaGenerateResponse, + OllamaMessage, + OllamaTool, + OllamaToolCall, + OllamaUsageResponse, +} from "../../vendor-sdk-types/ollama"; +import { BasePlugin } from "../core"; +import type { AsyncEndOf } from "../core/channel-definitions"; +import { + traceAsyncChannel, + traceStreamingChannel, + unsubscribeAll, +} from "../core/channel-tracing"; +import { ollamaChannels } from "./ollama-channels"; + +export class OllamaPlugin extends BasePlugin { + protected onEnable(): void { + this.unsubscribers.push( + traceStreamingChannel(ollamaChannels.chat, { + name: "ollama.chat", + type: SpanTypeAttribute.LLM, + extractInput: extractOllamaChatInput, + extractOutput: (result, event) => + extractOllamaChatOutput( + result, + countOllamaToolCalls(event?.arguments?.[0]?.messages), + ), + extractMetadata: extractOllamaResponseMetadata, + extractMetrics: extractOllamaMetrics, + aggregateChunks: aggregateOllamaChatChunks, + }), + traceStreamingChannel(ollamaChannels.generate, { + name: "ollama.generate", + type: SpanTypeAttribute.LLM, + extractInput: extractOllamaGenerateInput, + extractOutput: extractOllamaGenerateOutput, + extractMetadata: extractOllamaResponseMetadata, + extractMetrics: extractOllamaMetrics, + aggregateChunks: aggregateOllamaGenerateChunks, + }), + traceAsyncChannel(ollamaChannels.embed, { + name: "ollama.embed", + type: SpanTypeAttribute.LLM, + extractInput: extractOllamaEmbedInput, + extractOutput: extractOllamaEmbedOutput, + extractMetadata: extractOllamaResponseMetadata, + extractMetrics: extractOllamaMetrics, + }), + ); + } + + protected onDisable(): void { + this.unsubscribers = unsubscribeAll(this.unsubscribers); + } +} + +function isNonNegativeNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0; +} + +function normalizeFinishReason( + response: { done_reason?: string } | undefined, + hasToolCalls = false, +): string { + if (hasToolCalls) { + return "tool_calls"; + } + return response?.done_reason || "stop"; +} + +function stringifyArguments(value: unknown): string { + if (typeof value === "string") { + return value; + } + try { + return JSON.stringify(value ?? {}); + } catch { + return String(value); + } +} + +function syntheticToolCallId(name: string, index: number): string { + const normalizedName = name.replace(/[^a-zA-Z0-9_-]/g, "_") || "tool"; + return `ollama_call_${normalizedName}_${index}`; +} + +function normalizeToolCall( + toolCall: OllamaToolCall, + index: number, +): Record | undefined { + const name = toolCall.function?.name; + if (typeof name !== "string" || name.length === 0) { + return undefined; + } + + return { + id: + typeof toolCall.id === "string" && toolCall.id.length > 0 + ? toolCall.id + : syntheticToolCallId(name, index), + type: "function", + function: { + name, + arguments: stringifyArguments(toolCall.function?.arguments), + }, + }; +} + +function normalizeToolCalls( + toolCalls: OllamaToolCall[] | undefined, + syntheticIdOffset = 0, +): Record[] { + if (!Array.isArray(toolCalls)) { + return []; + } + + return toolCalls.flatMap((toolCall, index) => { + const normalized = normalizeToolCall(toolCall, syntheticIdOffset + index); + return normalized ? [normalized] : []; + }); +} + +function countOllamaToolCalls(messages: unknown): number { + if (!Array.isArray(messages)) { + return 0; + } + + return messages.reduce( + (count, message) => + count + + (isObject(message) && Array.isArray(message.tool_calls) + ? message.tool_calls.length + : 0), + 0, + ); +} + +function imageBytes(value: unknown): Uint8Array | undefined { + if (value instanceof Uint8Array) { + return value; + } + if (value instanceof ArrayBuffer) { + return new Uint8Array(value); + } + if (typeof value !== "string" || value.startsWith("data:")) { + return undefined; + } + try { + const decoded = atob(value.slice(0, 24)); + return Uint8Array.from(decoded, (character) => character.charCodeAt(0)); + } catch { + return undefined; + } +} + +function inferImageMediaType(value: unknown): string | undefined { + if (typeof value === "string") { + const dataUrlType = value.match(/^data:(image\/[^;]+);base64,/i)?.[1]; + if (dataUrlType) { + return dataUrlType; + } + } + + const bytes = imageBytes(value); + if (!bytes) { + return undefined; + } + if ( + bytes[0] === 0x89 && + bytes[1] === 0x50 && + bytes[2] === 0x4e && + bytes[3] === 0x47 + ) { + return "image/png"; + } + if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) { + return "image/jpeg"; + } + const signature = String.fromCharCode(...bytes.slice(0, 12)); + if (signature.startsWith("GIF87a") || signature.startsWith("GIF89a")) { + return "image/gif"; + } + if (signature.startsWith("RIFF") && signature.slice(8, 12) === "WEBP") { + return "image/webp"; + } + return undefined; +} + +function normalizeTextAndImages( + content: string | null, + images: unknown[] | undefined, +): { + content: unknown; + unrecognizedImages?: unknown[]; +} { + if (!images || images.length === 0) { + return { content }; + } + + const imageParts: Record[] = []; + const unrecognizedImages: unknown[] = []; + for (const image of images) { + let localImagePath: string | undefined; + let localPathMediaType: string | undefined; + if ( + typeof image === "string" && + !/^data:/i.test(image) && + !/^https?:\/\//i.test(image) + ) { + const extension = image.match(/\.([a-z0-9]+)$/i)?.[1]?.toLowerCase(); + const extensionMediaType = { + png: "image/png", + jpg: "image/jpeg", + jpeg: "image/jpeg", + gif: "image/gif", + webp: "image/webp", + }[extension ?? ""]; + if (extensionMediaType && iso.statSync) { + try { + if (iso.statSync(image).isFile()) { + localImagePath = image; + localPathMediaType = extensionMediaType; + } + } catch { + // Ollama treats unreadable strings as base64, so preserve them below. + } + } + } + + const mediaType = localPathMediaType ?? inferImageMediaType(image); + if (!mediaType) { + unrecognizedImages.push(image); + continue; + } + const processedImage = localImagePath + ? new Attachment({ + data: localImagePath, + filename: localImagePath.split(/[\\/]/).at(-1) ?? "image", + contentType: mediaType, + }) + : processInputAttachments({ + type: "image", + image, + mediaType, + }).image; + imageParts.push({ + type: "image_url", + image_url: { url: processedImage }, + }); + } + + return { + content: + imageParts.length > 0 + ? [...(content ? [{ type: "text", text: content }] : []), ...imageParts] + : content, + ...(unrecognizedImages.length > 0 ? { unrecognizedImages } : {}), + }; +} + +function normalizeMessage( + message: OllamaMessage, + toolCallId?: string, + syntheticToolCallIdOffset = 0, +): Record | undefined { + if (typeof message.role !== "string") { + return undefined; + } + + const toolCalls = normalizeToolCalls( + message.tool_calls, + syntheticToolCallIdOffset, + ); + if (message.role === "tool") { + const toolName = + typeof message.tool_name === "string" && message.tool_name.length > 0 + ? message.tool_name + : "tool"; + return { + role: "tool", + tool_call_id: toolCallId ?? syntheticToolCallId(toolName, 0), + content: typeof message.content === "string" ? message.content : "", + }; + } + + const normalizedContent = normalizeTextAndImages( + typeof message.content === "string" + ? message.content || (toolCalls.length > 0 ? null : "") + : toolCalls.length > 0 + ? null + : "", + message.images, + ); + return { + role: message.role, + content: normalizedContent.content, + ...(typeof message.thinking === "string" && message.thinking.length > 0 + ? { reasoning: message.thinking } + : {}), + ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}), + ...(normalizedContent.unrecognizedImages + ? { images: normalizedContent.unrecognizedImages } + : {}), + }; +} + +function normalizeMessages(messages: unknown): Record[] { + if (!Array.isArray(messages)) { + return []; + } + + const pendingToolCallIds = new Map(); + let syntheticToolCallIdOffset = 0; + return messages.flatMap((message) => { + if (!isObject(message)) { + return []; + } + + const ollamaMessage = message as OllamaMessage; + let toolCallId: string | undefined; + if (ollamaMessage.role === "tool") { + const toolName = ollamaMessage.tool_name; + if (typeof toolName === "string") { + toolCallId = pendingToolCallIds.get(toolName)?.shift(); + } + } + + const normalized = normalizeMessage( + ollamaMessage, + toolCallId, + syntheticToolCallIdOffset, + ); + if (Array.isArray(ollamaMessage.tool_calls)) { + syntheticToolCallIdOffset += ollamaMessage.tool_calls.length; + } + if (ollamaMessage.role === "assistant" && normalized) { + const toolCalls = Array.isArray(normalized.tool_calls) + ? normalized.tool_calls + : []; + for (const toolCall of toolCalls) { + if (!isObject(toolCall) || !isObject(toolCall.function)) { + continue; + } + const name = toolCall.function.name; + const id = toolCall.id; + if (typeof name === "string" && typeof id === "string") { + const ids = pendingToolCallIds.get(name) ?? []; + ids.push(id); + pendingToolCallIds.set(name, ids); + } + } + } + return normalized ? [normalized] : []; + }); +} + +function normalizeToolDefinition( + tool: OllamaTool, +): Record | undefined { + const fn = tool.function; + if (!fn || typeof fn.name !== "string" || fn.name.length === 0) { + return undefined; + } + + return { + type: "function", + function: { + name: fn.name, + ...(typeof fn.description === "string" + ? { description: fn.description } + : {}), + ...(isObject(fn.parameters) ? { parameters: fn.parameters } : {}), + }, + }; +} + +function extractToolsMetadata(tools: unknown): Record { + if (!Array.isArray(tools)) { + return {}; + } + const normalized = tools.flatMap((tool) => { + if (!isObject(tool)) { + return []; + } + const definition = normalizeToolDefinition(tool as OllamaTool); + return definition ? [definition] : []; + }); + return normalized.length > 0 ? { tools: normalized } : {}; +} + +function extractOptionsMetadata(options: unknown): Record { + if (!isObject(options)) { + return {}; + } + + const metadata: Record = {}; + const mappings = [ + ["temperature", "temperature"], + ["top_p", "top_p"], + ["num_predict", "max_tokens"], + ["frequency_penalty", "frequency_penalty"], + ["presence_penalty", "presence_penalty"], + ["stop", "stop"], + ] as const; + for (const [source, target] of mappings) { + if (options[source] !== undefined) { + metadata[target] = options[source]; + } + } + return metadata; +} + +function extractRequestMetadata( + request: + | { + format?: unknown; + model?: unknown; + options?: unknown; + } + | undefined, +): Record { + return { + provider: "ollama", + ...(typeof request?.model === "string" ? { model: request.model } : {}), + ...extractOptionsMetadata(request?.options), + ...(request?.format !== undefined + ? { response_format: request.format } + : {}), + }; +} + +export function extractOllamaChatInput([request]: [ + OllamaChatRequest, + ...unknown[], +]): { + input: unknown; + metadata: Record; +} { + return { + input: normalizeMessages(request.messages), + metadata: { + ...extractRequestMetadata(request), + ...extractToolsMetadata(request.tools), + }, + }; +} + +export function extractOllamaGenerateInput([request]: [ + OllamaGenerateRequest, + ...unknown[], +]): { + input: unknown; + metadata: Record; +} { + const normalizedPrompt = normalizeTextAndImages( + typeof request.prompt === "string" ? request.prompt : "", + Array.isArray(request.images) ? request.images : undefined, + ); + const input = [ + ...(typeof request.system === "string" + ? [{ role: "system", content: request.system }] + : []), + { + role: "user", + content: normalizedPrompt.content, + ...(typeof request.suffix === "string" ? { suffix: request.suffix } : {}), + ...(normalizedPrompt.unrecognizedImages + ? { images: normalizedPrompt.unrecognizedImages } + : {}), + }, + ]; + + return { + input, + metadata: extractRequestMetadata(request), + }; +} + +function extractOllamaEmbedInput([request]: [ + OllamaEmbedRequest, + ...unknown[], +]): { + input: unknown; + metadata: Record; +} { + return { + input: request.input, + metadata: extractRequestMetadata(request), + }; +} + +export function extractOllamaChatOutput( + result: OllamaChatResponse, + syntheticToolCallIdOffset = 0, +): unknown { + if (!isObject(result) || !isObject(result.message)) { + return undefined; + } + + const message = normalizeMessage( + result.message as OllamaMessage, + undefined, + syntheticToolCallIdOffset, + ); + if (!message) { + return undefined; + } + const toolCalls = Array.isArray(message.tool_calls) ? message.tool_calls : []; + + return [ + { + index: 0, + finish_reason: normalizeFinishReason(result, toolCalls.length > 0), + message, + }, + ]; +} + +function extractOllamaGenerateOutput(result: OllamaGenerateResponse): unknown { + if (!isObject(result) || typeof result.response !== "string") { + return undefined; + } + + return [ + { + index: 0, + finish_reason: normalizeFinishReason(result), + message: { + role: "assistant", + content: result.response, + ...(typeof result.thinking === "string" && result.thinking.length > 0 + ? { reasoning: result.thinking } + : {}), + }, + }, + ]; +} + +export function extractOllamaEmbedOutput(result: OllamaEmbedResponse): unknown { + const embedding = Array.isArray(result?.embeddings) + ? result.embeddings[0] + : undefined; + return Array.isArray(embedding) + ? { embedding_length: embedding.length } + : undefined; +} + +function extractOllamaResponseMetadata( + result: OllamaUsageResponse, +): Record | undefined { + return typeof result?.model === "string" + ? { model: result.model } + : undefined; +} + +export function extractOllamaMetrics( + result: OllamaUsageResponse, +): Record { + const metrics: Record = {}; + const promptTokens = result?.prompt_eval_count; + const completionTokens = result?.eval_count; + + if (isNonNegativeNumber(promptTokens)) { + metrics.prompt_tokens = promptTokens; + } + if (isNonNegativeNumber(completionTokens)) { + metrics.completion_tokens = completionTokens; + } + if ( + isNonNegativeNumber(promptTokens) || + isNonNegativeNumber(completionTokens) + ) { + metrics.tokens = + (isNonNegativeNumber(promptTokens) ? promptTokens : 0) + + (isNonNegativeNumber(completionTokens) ? completionTokens : 0); + } + return metrics; +} + +export function aggregateOllamaChatChunks( + chunks: OllamaChatResponse[], + _result?: unknown, + event?: AsyncEndOf, + _startTime?: number, +): { + output: unknown; + metrics: Record; + metadata?: Record; +} { + const last = chunks.at(-1); + const message: OllamaMessage = { + role: "assistant", + content: chunks.map((chunk) => chunk.message?.content ?? "").join(""), + thinking: chunks.map((chunk) => chunk.message?.thinking ?? "").join(""), + tool_calls: chunks.flatMap((chunk) => chunk.message?.tool_calls ?? []), + }; + const response: OllamaChatResponse = { + ...last, + message, + }; + return { + output: extractOllamaChatOutput( + response, + countOllamaToolCalls(event?.arguments?.[0]?.messages), + ), + metrics: extractOllamaMetrics(last ?? {}), + metadata: extractOllamaResponseMetadata(last ?? {}), + }; +} + +export function aggregateOllamaGenerateChunks( + chunks: OllamaGenerateResponse[], + _result?: unknown, + _event?: unknown, + _startTime?: number, +): { + output: unknown; + metrics: Record; + metadata?: Record; +} { + const last = chunks.at(-1); + const response: OllamaGenerateResponse = { + ...last, + response: chunks.map((chunk) => chunk.response ?? "").join(""), + thinking: chunks.map((chunk) => chunk.thinking ?? "").join(""), + }; + return { + output: extractOllamaGenerateOutput(response), + metrics: extractOllamaMetrics(last ?? {}), + metadata: extractOllamaResponseMetadata(last ?? {}), + }; +} diff --git a/js/src/span-origin.ts b/js/src/span-origin.ts index 0c601c027..df5fc2844 100644 --- a/js/src/span-origin.ts +++ b/js/src/span-origin.ts @@ -30,6 +30,7 @@ export const INSTRUMENTATION_NAMES = { LANGSMITH: "langsmith", MASTRA: "mastra", MISTRAL: "mistral", + OLLAMA: "ollama", OPENAI: "openai", OPENAI_AGENTS: "openai-agents", OPENAI_CODEX: "openai-codex", diff --git a/js/src/vendor-sdk-types/ollama.ts b/js/src/vendor-sdk-types/ollama.ts new file mode 100644 index 000000000..d3626bf8f --- /dev/null +++ b/js/src/vendor-sdk-types/ollama.ts @@ -0,0 +1,105 @@ +// ollama types + +export interface OllamaOptions { + num_predict?: number; + temperature?: number; + top_k?: number; + top_p?: number; + frequency_penalty?: number; + presence_penalty?: number; + stop?: string[]; +} + +export interface OllamaToolCall { + id?: string; + function?: { + name?: string; + arguments?: unknown; + }; +} + +export interface OllamaTool { + type?: string; + function?: { + name?: string; + description?: string; + parameters?: Record; + }; +} + +export interface OllamaMessage { + role?: string; + content?: string; + thinking?: string; + images?: unknown[]; + tool_calls?: OllamaToolCall[]; + tool_name?: string; +} + +export interface OllamaChatRequest { + model: string; + messages?: OllamaMessage[]; + stream?: boolean; + format?: string | Record; + tools?: OllamaTool[]; + options?: OllamaOptions; +} + +export interface OllamaGenerateRequest { + model: string; + prompt: string; + suffix?: string; + system?: string; + stream?: boolean; + format?: string | Record; + images?: unknown[]; + options?: OllamaOptions; +} + +export interface OllamaEmbedRequest { + model: string; + input: string | string[]; + dimensions?: number; + options?: OllamaOptions; +} + +export interface OllamaUsageResponse { + model?: string; + prompt_eval_count?: number; + eval_count?: number; +} + +export interface OllamaChatResponse extends OllamaUsageResponse { + message?: OllamaMessage; + done?: boolean; + done_reason?: string; +} + +export interface OllamaGenerateResponse extends OllamaUsageResponse { + response?: string; + thinking?: string; + done?: boolean; + done_reason?: string; +} + +export interface OllamaEmbedResponse extends OllamaUsageResponse { + embeddings?: number[][]; +} + +export interface OllamaAsyncIterator extends AsyncIterable { + abort?: () => void; +} + +export type OllamaChatResult = + | OllamaChatResponse + | OllamaAsyncIterator; + +export type OllamaGenerateResult = + | OllamaGenerateResponse + | OllamaAsyncIterator; + +export interface OllamaClient { + chat?: (request: OllamaChatRequest) => Promise; + generate?: (request: OllamaGenerateRequest) => Promise; + embed?: (request: OllamaEmbedRequest) => Promise; +} diff --git a/js/src/wrappers/ollama.test.ts b/js/src/wrappers/ollama.test.ts new file mode 100644 index 000000000..9a8ebff46 --- /dev/null +++ b/js/src/wrappers/ollama.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it, vi } from "vitest"; +import { debugLogger } from "../debug-logger"; +import { ollamaChannels } from "../instrumentation/plugins/ollama-channels"; +import type { OllamaClient } from "../vendor-sdk-types/ollama"; +import { wrapOllama } from "./ollama"; + +describe("wrapOllama", () => { + it("emits channel events for every supported generation surface", async () => { + const client: OllamaClient = { + chat: vi.fn(async () => ({ + message: { role: "assistant", content: "OK" }, + done: true, + })), + generate: vi.fn(async () => ({ response: "OK", done: true })), + embed: vi.fn(async () => ({ embeddings: [[0.1, 0.2]] })), + }; + const chatSpy = vi + .spyOn(ollamaChannels.chat, "tracePromise") + .mockImplementation((fn) => fn()); + const generateSpy = vi + .spyOn(ollamaChannels.generate, "tracePromise") + .mockImplementation((fn) => fn()); + const embedSpy = vi + .spyOn(ollamaChannels.embed, "tracePromise") + .mockImplementation((fn) => fn()); + const wrapped = wrapOllama(client); + expect(wrapped.chat).toBe(wrapped.chat); + expect(wrapped.generate).toBe(wrapped.generate); + expect(wrapped.embed).toBe(wrapped.embed); + await wrapped.chat?.({ + model: "gpt-oss:20b", + messages: [{ role: "user", content: "Say OK." }], + }); + await wrapped.generate?.({ + model: "gpt-oss:20b", + prompt: "Say OK.", + }); + await wrapped.embed?.({ model: "embeddinggemma", input: "hello" }); + + expect(chatSpy).toHaveBeenCalledOnce(); + expect(generateSpy).toHaveBeenCalledOnce(); + expect(embedSpy).toHaveBeenCalledOnce(); + expect(wrapOllama(client)).toBe(wrapped); + expect(wrapOllama(wrapped)).toBe(wrapped); + }); + + it("returns unsupported objects unchanged", () => { + const client = {}; + const warn = vi.spyOn(debugLogger, "warn").mockImplementation(() => {}); + + expect(wrapOllama(client)).toBe(client); + expect(warn).toHaveBeenCalledWith( + "Unsupported Ollama library. Not wrapping.", + ); + }); + + it("refreshes stable wrappers when client methods change", async () => { + const originalChat = vi.fn(async () => ({ + message: { role: "assistant", content: "original" }, + done: true, + })); + const client: OllamaClient = { chat: originalChat }; + const tracePromise = vi + .spyOn(ollamaChannels.chat, "tracePromise") + .mockImplementation((fn) => fn()); + const wrapped = wrapOllama(client); + const firstWrappedChat = wrapped.chat; + + expect(firstWrappedChat).toBe(wrapped.chat); + + const replacementChat = vi.fn(async () => ({ + message: { role: "assistant", content: "replacement" }, + done: true, + })); + client.chat = replacementChat; + + expect(wrapped.chat).not.toBe(firstWrappedChat); + expect(wrapped.chat).toBe(wrapped.chat); + await wrapped.chat?.({ model: "gpt-oss:20b", messages: [] }); + expect(originalChat).not.toHaveBeenCalled(); + expect(replacementChat).toHaveBeenCalledOnce(); + + client.chat = undefined; + expect(wrapped.chat).toBeUndefined(); + tracePromise.mockRestore(); + }); +}); diff --git a/js/src/wrappers/ollama.ts b/js/src/wrappers/ollama.ts new file mode 100644 index 000000000..3f90067a9 --- /dev/null +++ b/js/src/wrappers/ollama.ts @@ -0,0 +1,122 @@ +import { debugLogger } from "../debug-logger"; +import { ollamaChannels } from "../instrumentation/plugins/ollama-channels"; +import { isObject } from "../../util"; +import type { + OllamaChatRequest, + OllamaChatResult, + OllamaClient, + OllamaEmbedRequest, + OllamaEmbedResponse, + OllamaGenerateRequest, + OllamaGenerateResult, +} from "../vendor-sdk-types/ollama"; + +/** + * Wrap an Ollama client so generation and embedding calls emit Braintrust + * diagnostics-channel events. + */ +export function wrapOllama(ollama: T): T { + if (isSupportedOllamaClient(ollama)) { + return ollamaProxy(ollama) as T; + } + + debugLogger.warn("Unsupported Ollama library. Not wrapping."); + return ollama; +} + +const ollamaProxyCache = new WeakMap(); + +function isSupportedOllamaClient(value: unknown): value is OllamaClient { + return ( + isObject(value) && + ["chat", "generate", "embed"].some( + (name) => typeof value[name] === "function", + ) + ); +} + +function ollamaProxy(ollama: OllamaClient): OllamaClient { + const cached = ollamaProxyCache.get(ollama); + if (cached) { + return cached; + } + + let chatSource: OllamaClient["chat"]; + let wrappedChat: OllamaClient["chat"]; + let generateSource: OllamaClient["generate"]; + let wrappedGenerate: OllamaClient["generate"]; + let embedSource: OllamaClient["embed"]; + let wrappedEmbed: OllamaClient["embed"]; + + const proxy = new Proxy(ollama, { + get(target, prop, receiver) { + switch (prop) { + case "chat": { + const source = target.chat; + if (source !== chatSource) { + chatSource = source; + wrappedChat = + typeof source === "function" + ? wrapChat(source.bind(target)) + : source; + } + return wrappedChat; + } + case "generate": { + const source = target.generate; + if (source !== generateSource) { + generateSource = source; + wrappedGenerate = + typeof source === "function" + ? wrapGenerate(source.bind(target)) + : source; + } + return wrappedGenerate; + } + case "embed": { + const source = target.embed; + if (source !== embedSource) { + embedSource = source; + wrappedEmbed = + typeof source === "function" + ? wrapEmbed(source.bind(target)) + : source; + } + return wrappedEmbed; + } + default: + return Reflect.get(target, prop, receiver); + } + }, + }); + ollamaProxyCache.set(ollama, proxy); + ollamaProxyCache.set(proxy, proxy); + return proxy; +} + +function wrapChat( + chat: (request: OllamaChatRequest) => Promise, +): NonNullable { + return (request) => + ollamaChannels.chat.tracePromise(() => chat(request), { + arguments: [request], + }); +} + +function wrapGenerate( + generate: (request: OllamaGenerateRequest) => Promise, +): NonNullable { + return (request) => + ollamaChannels.generate.tracePromise(() => generate(request), { + arguments: [request], + }); +} + +function wrapEmbed( + embed: (request: OllamaEmbedRequest) => Promise, +): NonNullable { + return (request) => + ollamaChannels.embed.tracePromise(() => embed(request), { + arguments: [request], + }); +}