diff --git a/.github/actions/setup-python-env/action.yml b/.github/actions/setup-python-env/action.yml index f4132874..c0c3272d 100644 --- a/.github/actions/setup-python-env/action.yml +++ b/.github/actions/setup-python-env/action.yml @@ -9,8 +9,14 @@ inputs: runs: using: "composite" steps: + - name: Install Python build dependencies + if: runner.os == 'Linux' + shell: bash + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends --yes libbz2-dev liblzma-dev libreadline-dev - name: Set up mise - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 + uses: jdx/mise-action@7e36c90d9ab29c415a2384db3006f3ec8a8cc654 # v4.2.4 with: cache: true experimental: true diff --git a/.github/workflows/adk-py-test.yaml b/.github/workflows/adk-py-test.yaml index a03177f6..75692b20 100644 --- a/.github/workflows/adk-py-test.yaml +++ b/.github/workflows/adk-py-test.yaml @@ -13,10 +13,10 @@ jobs: timeout-minutes: 15 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up mise - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 + uses: jdx/mise-action@7e36c90d9ab29c415a2384db3006f3ec8a8cc654 # v4.2.4 with: cache: true experimental: true diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index 90f3b431..8bf10c09 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -13,11 +13,11 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 10 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 # Fetch full history for proper diff - name: Set up mise - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 + uses: jdx/mise-action@7e36c90d9ab29c415a2384db3006f3ec8a8cc654 # v4.2.4 with: cache: true experimental: true @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: bash scripts/ensure-pinned-actions.sh static_checks: @@ -40,7 +40,7 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Python environment uses: ./.github/actions/setup-python-env with: @@ -61,7 +61,7 @@ jobs: os: [ubuntu-24.04, windows-2025] steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Python environment uses: ./.github/actions/setup-python-env with: @@ -87,7 +87,7 @@ jobs: shard: [0, 1, 2, 3, 4, 5] steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Python environment uses: ./.github/actions/setup-python-env with: @@ -150,9 +150,9 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 10 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up mise - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 + uses: jdx/mise-action@7e36c90d9ab29c415a2384db3006f3ec8a8cc654 # v4.2.4 with: cache: true experimental: true diff --git a/.github/workflows/dependency-updates.yml b/.github/workflows/dependency-updates.yml index 5aae021c..ed96503a 100644 --- a/.github/workflows/dependency-updates.yml +++ b/.github/workflows/dependency-updates.yml @@ -12,9 +12,9 @@ jobs: update: runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up mise - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 + uses: jdx/mise-action@7e36c90d9ab29c415a2384db3006f3ec8a8cc654 # v4.2.4 with: cache: true experimental: true diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index e000445b..a3e26860 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -28,7 +28,7 @@ jobs: permission-pull-requests: write - name: Checkout parent repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: braintrustdata/braintrust path: braintrust diff --git a/.github/workflows/langchain-py-test.yaml b/.github/workflows/langchain-py-test.yaml index 7fba7906..f435af99 100644 --- a/.github/workflows/langchain-py-test.yaml +++ b/.github/workflows/langchain-py-test.yaml @@ -9,10 +9,10 @@ jobs: timeout-minutes: 15 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up mise - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 + uses: jdx/mise-action@7e36c90d9ab29c415a2384db3006f3ec8a8cc654 # v4.2.4 with: cache: true experimental: true diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index af36f794..9d4073ff 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -20,7 +20,7 @@ jobs: timeout-minutes: 10 steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 diff --git a/.github/workflows/publish-py-sdk.yaml b/.github/workflows/publish-py-sdk.yaml index be883f44..c0240280 100644 --- a/.github/workflows/publish-py-sdk.yaml +++ b/.github/workflows/publish-py-sdk.yaml @@ -45,12 +45,12 @@ jobs: release_type: ${{ steps.validate.outputs.release_type }} version: ${{ steps.validate.outputs.version }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.merge_commit_sha || (github.event_name == 'workflow_dispatch' && github.event.inputs.ref || github.ref) }} fetch-depth: 0 - name: Set up mise - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 + uses: jdx/mise-action@7e36c90d9ab29c415a2384db3006f3ec8a8cc654 # v4.2.4 with: cache: true experimental: true @@ -86,12 +86,12 @@ jobs: VERSION: ${{ needs.validate.outputs.version }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ env.COMMIT_SHA }} fetch-depth: 0 - name: Set up mise - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 + uses: jdx/mise-action@7e36c90d9ab29c415a2384db3006f3ec8a8cc654 # v4.2.4 with: cache: true experimental: true @@ -108,7 +108,7 @@ jobs: path: py/dist/ retention-days: 5 - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 with: packages-dir: py/dist/ @@ -163,12 +163,12 @@ jobs: VERSION: ${{ needs.validate.outputs.version }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ env.COMMIT_SHA }} fetch-depth: 0 - name: Set up mise - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 + uses: jdx/mise-action@7e36c90d9ab29c415a2384db3006f3ec8a8cc654 # v4.2.4 with: cache: true experimental: true @@ -186,7 +186,7 @@ jobs: retention-days: 5 - name: Publish to PyPI if: env.DRY_RUN != 'true' - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 with: packages-dir: py/dist/ @@ -237,7 +237,7 @@ jobs: timeout-minutes: 5 steps: - name: Post to Slack on success - uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3 + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 with: method: chat.postMessage token: ${{ secrets.SLACK_BOT_TOKEN }} @@ -261,7 +261,7 @@ jobs: timeout-minutes: 5 steps: - name: Post to Slack on failure - uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3 + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 with: method: chat.postMessage token: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/test-publish-py-sdk.yaml b/.github/workflows/test-publish-py-sdk.yaml index f9c60706..60194458 100644 --- a/.github/workflows/test-publish-py-sdk.yaml +++ b/.github/workflows/test-publish-py-sdk.yaml @@ -37,12 +37,12 @@ jobs: target_branch: ${{ steps.validate.outputs.target_branch }} version: ${{ steps.validate.outputs.version }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.inputs.ref }} fetch-depth: 0 - name: Set up mise - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 + uses: jdx/mise-action@7e36c90d9ab29c415a2384db3006f3ec8a8cc654 # v4.2.4 with: cache: true experimental: true @@ -86,12 +86,12 @@ jobs: VERSION: ${{ needs.validate.outputs.version }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ env.COMMIT_SHA }} fetch-depth: 0 - name: Set up mise - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 + uses: jdx/mise-action@7e36c90d9ab29c415a2384db3006f3ec8a8cc654 # v4.2.4 with: cache: true experimental: true @@ -155,7 +155,7 @@ jobs: retention-days: 5 - name: Publish to TestPyPI if: env.DRY_RUN != 'true' && (env.RELEASE_TYPE != 'canary' || (steps.should_publish.outputs.should_publish == 'true' && steps.ci_status.outputs.should_publish == 'true')) - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 with: repository-url: https://test.pypi.org/legacy/ packages-dir: py/dist/ @@ -226,7 +226,7 @@ jobs: steps: - name: Post to Slack on dry run success if: needs.validate.outputs.dry_run == 'true' - uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3 + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 with: method: chat.postMessage token: ${{ secrets.SLACK_BOT_TOKEN }} @@ -244,7 +244,7 @@ jobs: text: "${{ format('*Mode:* dry run\n*Release type:* {0}\n*Version:* {1}\n*Ref:* {2}\n*Commit:* {3}\n\n<{4}/{5}/actions/runs/{6}|View Run>', needs.validate.outputs.release_type, needs.validate.outputs.version, github.event.inputs.ref, needs.validate.outputs.commit_sha, github.server_url, github.repository, github.run_id) }}" - name: Post to Slack on prerelease success if: needs.validate.outputs.dry_run != 'true' && needs.validate.outputs.release_type == 'prerelease' - uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3 + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 with: method: chat.postMessage token: ${{ secrets.SLACK_BOT_TOKEN }} @@ -262,7 +262,7 @@ jobs: text: "${{ format('*Version:* {0}\n*Ref:* {1}\n*Install:* `pip install -i https://test.pypi.org/simple/ braintrust=={0}`\n*Package:* \n\n<{2}/{3}/actions/runs/{4}|View Run>', needs.build-and-publish.outputs.version, github.event.inputs.ref, github.server_url, github.repository, github.run_id) }}" - name: Post to Slack on canary success if: needs.validate.outputs.dry_run != 'true' && needs.validate.outputs.release_type == 'canary' - uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3 + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 with: method: chat.postMessage token: ${{ secrets.SLACK_BOT_TOKEN }} @@ -286,7 +286,7 @@ jobs: timeout-minutes: 5 steps: - name: Post to Slack on intentional skip - uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3 + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 with: method: chat.postMessage token: ${{ secrets.SLACK_BOT_TOKEN }} @@ -310,7 +310,7 @@ jobs: timeout-minutes: 5 steps: - name: Post to Slack on failure - uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3 + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 with: method: chat.postMessage token: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/update-session-weights.yaml b/.github/workflows/update-session-weights.yaml index 7338f791..e684c439 100644 --- a/.github/workflows/update-session-weights.yaml +++ b/.github/workflows/update-session-weights.yaml @@ -21,7 +21,7 @@ jobs: shard: [0, 1, 2, 3] steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Python environment uses: ./.github/actions/setup-python-env with: @@ -44,7 +44,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Download measured durations uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: diff --git a/py/noxfile.py b/py/noxfile.py index 60c44805..4e44f45a 100644 --- a/py/noxfile.py +++ b/py/noxfile.py @@ -218,6 +218,21 @@ def _pinned_python_version(): # Version matrices — derived from dependency groups in pyproject.toml # --------------------------------------------------------------------------- +AI_SDK_VERSIONS = _get_matrix_versions("ai-sdk") + + +@nox.session() +@nox.parametrize("version", AI_SDK_VERSIONS, ids=AI_SDK_VERSIONS) +def test_ai_sdk(session, version): + """Test the Vercel AI SDK for Python telemetry integration.""" + if sys.version_info < (3, 12): + session.skip("Vercel AI SDK for Python requires Python 3.12+") + _install_test_deps(session) + _install_matrix_dep(session, "openai", LATEST) + _install_matrix_dep(session, "ai-sdk", version) + _run_tests(session, f"{INTEGRATION_DIR}/ai_sdk/test_ai_sdk.py", version=version) + + ANTHROPIC_VERSIONS = _get_matrix_versions("anthropic") @@ -757,24 +772,28 @@ def pylint(session): def _install_test_deps(session): # Choose the way we'll install braintrust ... wheel or source. install_wheel = "--wheel" in session.posargs - bt = _get_braintrust_wheel() if install_wheel else "." - # Install braintrust itself (wheel or editable source). - session.install(bt) + # Install braintrust itself. Source installs are editable so that + # site-packages resolves to src/ instead of holding a copy. A copy goes + # stale as soon as the session venv is reused (``nox -R``), and anything + # that imports braintrust outside of pytest -- notably the subprocesses + # spawned by ``verify_autoinstrument_script`` -- picks up site-packages + # rather than the source tree, so it would silently exercise the code from + # whenever the venv was last built. + session.install(*([_get_braintrust_wheel()] if install_wheel else ["-e", "."])) # Install base test deps (pytest, pytest-asyncio, pytest-vcr) from the # lockfile so transitive deps are pinned and reproducible. _install_group_locked(session, "test") - # Sanity check we have installed braintrust (and that it is from a wheel if needed) - session.run("python", "-c", "import braintrust") - if install_wheel: - lines = [ - "import sys, braintrust as b", - "print(f'Using braintrust from: {b.__file__}')", - "sys.exit(0 if 'site-packages' in b.__file__ else 1)", - ] - session.run("python", "-c", ";".join(lines)) + # Sanity check braintrust imports from where this mode expects it: + # site-packages for a wheel, the source tree for an editable install. + lines = [ + "import sys, braintrust as b", + "print(f'Using braintrust from: {b.__file__}')", + f"sys.exit(0 if {install_wheel} == ('site-packages' in b.__file__) else 1)", + ] + session.run("python", "-c", ";".join(lines)) def _get_braintrust_wheel(): diff --git a/py/pyproject.toml b/py/pyproject.toml index 0787f36a..3c1098c5 100644 --- a/py/pyproject.toml +++ b/py/pyproject.toml @@ -237,6 +237,7 @@ lint = [ # can find mutually compatible transitive deps. "agentscope<2", "agno", + "ai>=0.4.0; python_version>='3.12'", "anthropic", "autogen-agentchat", "autogen-ext[openai]", @@ -340,6 +341,10 @@ conflicts = [ [tool.braintrust.matrix] +[tool.braintrust.matrix.ai-sdk] +latest = "ai[openai]==0.4.2" +"0.4.0" = "ai[openai]==0.4.0" + [tool.braintrust.matrix.openai] latest = "openai==2.53.0" "1.92.0" = "openai==1.92.0" @@ -504,6 +509,7 @@ latest = "botocore==1.43.63" # --------------------------------------------------------------------------- [tool.braintrust.cassette-dirs] adk = ["google-adk"] +ai_sdk = ["ai-sdk"] agentscope = ["agentscope"] agno = ["agno"] autogen = ["autogen-agentchat"] @@ -529,6 +535,7 @@ pydantic_ai = ["pydantic-ai-integration", "pydantic-ai-wrap-openai"] strands = ["strands-agents"] [tool.braintrust.vendor-packages] +ai-sdk = "ai" agno = "agno" agentscope = "agentscope" autogen-agentchat = "autogen_agentchat" diff --git a/py/src/braintrust/__init__.py b/py/src/braintrust/__init__.py index 89e88d0c..1e66fced 100644 --- a/py/src/braintrust/__init__.py +++ b/py/src/braintrust/__init__.py @@ -68,6 +68,7 @@ def is_equal(expected, output): from .functions.invoke import * from .functions.stream import * from .generated_types import * +from .integrations.ai_sdk import setup_ai_sdk as setup_ai_sdk from .integrations.anthropic import wrap_anthropic as wrap_anthropic from .integrations.instructor import wrap_instructor as wrap_instructor from .integrations.litellm import wrap_litellm as wrap_litellm diff --git a/py/src/braintrust/auto.py b/py/src/braintrust/auto.py index b873fe6d..beecc64f 100644 --- a/py/src/braintrust/auto.py +++ b/py/src/braintrust/auto.py @@ -11,6 +11,7 @@ ADKIntegration, AgentScopeIntegration, AgnoIntegration, + AISDKIntegration, AnthropicIntegration, AutoGenIntegration, BedrockRuntimeIntegration, @@ -58,6 +59,7 @@ def auto_instrument( openai: bool = True, anthropic: bool = True, litellm: bool = True, + ai_sdk: bool = True, pydantic_ai: bool = True, google_genai: bool = True, instructor: bool = True, @@ -94,6 +96,7 @@ def auto_instrument( openai: Enable OpenAI instrumentation (default: True) anthropic: Enable Anthropic instrumentation (default: True) litellm: Enable LiteLLM instrumentation (default: True) + ai_sdk: Enable Vercel AI SDK for Python instrumentation (default: True) pydantic_ai: Enable Pydantic AI instrumentation (default: True) google_genai: Enable Google GenAI instrumentation (default: True) instructor: Enable Instructor (structured-output) instrumentation (default: True) @@ -169,6 +172,8 @@ def auto_instrument( results["anthropic"] = _instrument_integration(AnthropicIntegration) if litellm: results["litellm"] = _instrument_integration(LiteLLMIntegration) + if ai_sdk: + results["ai_sdk"] = _instrument_integration(AISDKIntegration) if pydantic_ai: results["pydantic_ai"] = _instrument_integration(PydanticAIIntegration) if google_genai: diff --git a/py/src/braintrust/integrations/__init__.py b/py/src/braintrust/integrations/__init__.py index c9b69d9a..dc53a2d0 100644 --- a/py/src/braintrust/integrations/__init__.py +++ b/py/src/braintrust/integrations/__init__.py @@ -1,6 +1,7 @@ from .adk import ADKIntegration from .agentscope import AgentScopeIntegration from .agno import AgnoIntegration +from .ai_sdk import AISDKIntegration from .anthropic import AnthropicIntegration from .autogen import AutoGenIntegration from .bedrock_runtime import BedrockRuntimeIntegration @@ -29,6 +30,7 @@ "ADKIntegration", "AgentScopeIntegration", "AgnoIntegration", + "AISDKIntegration", "AnthropicIntegration", "AutoGenIntegration", "BedrockRuntimeIntegration", diff --git a/py/src/braintrust/integrations/ai_sdk/__init__.py b/py/src/braintrust/integrations/ai_sdk/__init__.py new file mode 100644 index 00000000..3c786a9d --- /dev/null +++ b/py/src/braintrust/integrations/ai_sdk/__init__.py @@ -0,0 +1,37 @@ +"""Braintrust integration for Vercel AI SDK for Python.""" + +from braintrust.logger import NOOP_SPAN, current_span, init_logger + +from .integration import AISDKIntegration +from .patchers import unregister_adapter +from .tracing import BraintrustAISDKAdapter + + +__all__ = [ + "AISDKIntegration", + "BraintrustAISDKAdapter", + "patch_ai_sdk", + "setup_ai_sdk", + "unpatch_ai_sdk", +] + + +def setup_ai_sdk( + api_key: str | None = None, + project_id: str | None = None, + project_name: str | None = None, +) -> bool: + """Initialize Braintrust and register the Vercel AI SDK telemetry adapter.""" + if current_span() == NOOP_SPAN: + init_logger(project=project_name, api_key=api_key, project_id=project_id) + return AISDKIntegration.setup() + + +def patch_ai_sdk() -> bool: + """Register the telemetry adapter without initializing a Braintrust logger.""" + return AISDKIntegration.setup() + + +def unpatch_ai_sdk() -> bool: + """Unregister the Braintrust telemetry adapter. Safe to call repeatedly.""" + return unregister_adapter() diff --git a/py/src/braintrust/integrations/ai_sdk/cassettes/0.4.0/test_agent_tool_loop_emits_run_model_and_tool_hierarchy.yaml b/py/src/braintrust/integrations/ai_sdk/cassettes/0.4.0/test_agent_tool_loop_emits_run_model_and_tool_hierarchy.yaml new file mode 100644 index 00000000..1d7cec2b --- /dev/null +++ b/py/src/braintrust/integrations/ai_sdk/cassettes/0.4.0/test_agent_tool_loop_emits_run_model_and_tool_hierarchy.yaml @@ -0,0 +1,244 @@ +interactions: +- request: + body: '{"input":[{"role":"system","content":"Always use get_temperature once before + answering. Be concise."},{"role":"user","content":[{"type":"input_text","text":"What + is the temperature in Paris?"}]}],"model":"gpt-4o-mini","stream":true,"tools":[{"type":"function","name":"get_temperature","description":"Get + the current temperature for a city.","parameters":{"properties":{"city":{"title":"City","type":"string"}},"required":["city"],"title":"get_temperature_Args","type":"object"}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '480' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 2.53.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.53.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: 'event: response.created + + data: {"type":"response.created","response":{"id":"resp_0271f8b317d34f1a006a73982a8c3c819190ba43ceb295bb6c","object":"response","created_at":1785960490,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[{"type":"function","description":"Get + the current temperature for a city.","name":"get_temperature","output_schema":null,"parameters":{"properties":{"city":{"title":"City","type":"string"}},"required":["city"],"title":"get_temperature_Args","type":"object","additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + + event: response.in_progress + + data: {"type":"response.in_progress","response":{"id":"resp_0271f8b317d34f1a006a73982a8c3c819190ba43ceb295bb6c","object":"response","created_at":1785960490,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[{"type":"function","description":"Get + the current temperature for a city.","name":"get_temperature","output_schema":null,"parameters":{"properties":{"city":{"title":"City","type":"string"}},"required":["city"],"title":"get_temperature_Args","type":"object","additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + + event: response.output_item.added + + data: {"type":"response.output_item.added","item":{"id":"fc_0271f8b317d34f1a006a73982b1ca8819184914bd063a45a49","type":"function_call","status":"in_progress","arguments":"","call_id":"call_ZcIlHRGQxEnMdBbZsbjZRowm","name":"get_temperature"},"output_index":0,"sequence_number":2} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"{\"","item_id":"fc_0271f8b317d34f1a006a73982b1ca8819184914bd063a45a49","obfuscation":"c4pPcHfnXSliDX","output_index":0,"sequence_number":3} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"city","item_id":"fc_0271f8b317d34f1a006a73982b1ca8819184914bd063a45a49","obfuscation":"Llosl0pRvNJC","output_index":0,"sequence_number":4} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"\":\"","item_id":"fc_0271f8b317d34f1a006a73982b1ca8819184914bd063a45a49","obfuscation":"qLUyNCHIkmicN","output_index":0,"sequence_number":5} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"Paris","item_id":"fc_0271f8b317d34f1a006a73982b1ca8819184914bd063a45a49","obfuscation":"l4Ver21lW93","output_index":0,"sequence_number":6} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"\"}","item_id":"fc_0271f8b317d34f1a006a73982b1ca8819184914bd063a45a49","obfuscation":"pZVGYstPDnCqlM","output_index":0,"sequence_number":7} + + + event: response.function_call_arguments.done + + data: {"type":"response.function_call_arguments.done","arguments":"{\"city\":\"Paris\"}","item_id":"fc_0271f8b317d34f1a006a73982b1ca8819184914bd063a45a49","output_index":0,"sequence_number":8} + + + event: response.output_item.done + + data: {"type":"response.output_item.done","item":{"id":"fc_0271f8b317d34f1a006a73982b1ca8819184914bd063a45a49","type":"function_call","status":"completed","arguments":"{\"city\":\"Paris\"}","call_id":"call_ZcIlHRGQxEnMdBbZsbjZRowm","name":"get_temperature"},"output_index":0,"sequence_number":9} + + + event: response.completed + + data: {"type":"response.completed","response":{"id":"resp_0271f8b317d34f1a006a73982a8c3c819190ba43ceb295bb6c","object":"response","created_at":1785960490,"status":"completed","background":false,"completed_at":1785960491,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[{"id":"fc_0271f8b317d34f1a006a73982b1ca8819184914bd063a45a49","type":"function_call","status":"completed","arguments":"{\"city\":\"Paris\"}","call_id":"call_ZcIlHRGQxEnMdBbZsbjZRowm","name":"get_temperature"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[{"type":"function","description":"Get + the current temperature for a city.","name":"get_temperature","output_schema":null,"parameters":{"properties":{"city":{"title":"City","type":"string"}},"required":["city"],"title":"get_temperature_Args","type":"object","additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":70,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":0},"output_tokens":15,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":85},"user":null,"metadata":{}},"sequence_number":10} + + + ' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a2686ea93c7178a5-YYZ + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Wed, 05 Aug 2026 20:08:10 GMT + openai-processing-ms: + - '184' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-request-id: + - req_fd26ebd2f9634bc1a85c5334d75c0f3e + status: + code: 200 + message: OK +- request: + body: '{"input":[{"role":"system","content":"Always use get_temperature once before + answering. Be concise."},{"role":"user","content":[{"type":"input_text","text":"What + is the temperature in Paris?"}]},{"type":"item_reference","id":"fc_0271f8b317d34f1a006a73982b1ca8819184914bd063a45a49"},{"type":"function_call_output","call_id":"call_ZcIlHRGQxEnMdBbZsbjZRowm","output":"{\"city\":\"Paris\",\"temperature\":21,\"unit\":\"celsius\"}"}],"model":"gpt-4o-mini","stream":true,"tools":[{"type":"function","name":"get_temperature","description":"Get + the current temperature for a city.","parameters":{"properties":{"city":{"title":"City","type":"string"}},"required":["city"],"title":"get_temperature_Args","type":"object"}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '713' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 2.53.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.53.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0271f8b317d34f1a006a73982b7d1c8191b3acc9be64742c71\",\"object\":\"response\",\"created_at\":1785960491,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Get + the current temperature for a city.\",\"name\":\"get_temperature\",\"output_schema\":null,\"parameters\":{\"properties\":{\"city\":{\"title\":\"City\",\"type\":\"string\"}},\"required\":[\"city\"],\"title\":\"get_temperature_Args\",\"type\":\"object\",\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}\n\nevent: + response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0271f8b317d34f1a006a73982b7d1c8191b3acc9be64742c71\",\"object\":\"response\",\"created_at\":1785960491,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Get + the current temperature for a city.\",\"name\":\"get_temperature\",\"output_schema\":null,\"parameters\":{\"properties\":{\"city\":{\"title\":\"City\",\"type\":\"string\"}},\"required\":[\"city\"],\"title\":\"get_temperature_Args\",\"type\":\"object\",\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}\n\nevent: + response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0271f8b317d34f1a006a73982c49988191a842610ba590d1d9\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}\n\nevent: + response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_0271f8b317d34f1a006a73982c49988191a842610ba590d1d9\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"The\",\"item_id\":\"msg_0271f8b317d34f1a006a73982c49988191a842610ba590d1d9\",\"logprobs\":[],\"obfuscation\":\"EWQYMOK6MbVtY\",\"output_index\":0,\"sequence_number\":4}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + temperature\",\"item_id\":\"msg_0271f8b317d34f1a006a73982c49988191a842610ba590d1d9\",\"logprobs\":[],\"obfuscation\":\"rdpg\",\"output_index\":0,\"sequence_number\":5}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + in\",\"item_id\":\"msg_0271f8b317d34f1a006a73982c49988191a842610ba590d1d9\",\"logprobs\":[],\"obfuscation\":\"Bhk1g9nUbWnrd\",\"output_index\":0,\"sequence_number\":6}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + Paris\",\"item_id\":\"msg_0271f8b317d34f1a006a73982c49988191a842610ba590d1d9\",\"logprobs\":[],\"obfuscation\":\"Wg2XAMcmoU\",\"output_index\":0,\"sequence_number\":7}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + is\",\"item_id\":\"msg_0271f8b317d34f1a006a73982c49988191a842610ba590d1d9\",\"logprobs\":[],\"obfuscation\":\"iKnxBBc1aV0WJ\",\"output_index\":0,\"sequence_number\":8}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + currently\",\"item_id\":\"msg_0271f8b317d34f1a006a73982c49988191a842610ba590d1d9\",\"logprobs\":[],\"obfuscation\":\"JnbrWm\",\"output_index\":0,\"sequence_number\":9}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + \",\"item_id\":\"msg_0271f8b317d34f1a006a73982c49988191a842610ba590d1d9\",\"logprobs\":[],\"obfuscation\":\"FBBjX41XOD8OZs3\",\"output_index\":0,\"sequence_number\":10}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"21\",\"item_id\":\"msg_0271f8b317d34f1a006a73982c49988191a842610ba590d1d9\",\"logprobs\":[],\"obfuscation\":\"fLKvnVKT3QxAVr\",\"output_index\":0,\"sequence_number\":11}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\xB0C\",\"item_id\":\"msg_0271f8b317d34f1a006a73982c49988191a842610ba590d1d9\",\"logprobs\":[],\"obfuscation\":\"QWeMAkba7mYinF\",\"output_index\":0,\"sequence_number\":12}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\".\",\"item_id\":\"msg_0271f8b317d34f1a006a73982c49988191a842610ba590d1d9\",\"logprobs\":[],\"obfuscation\":\"WVF4aR5TOr0MTNv\",\"output_index\":0,\"sequence_number\":13}\n\nevent: + response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0271f8b317d34f1a006a73982c49988191a842610ba590d1d9\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":14,\"text\":\"The + temperature in Paris is currently 21\xB0C.\"}\n\nevent: response.content_part.done\ndata: + {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_0271f8b317d34f1a006a73982c49988191a842610ba590d1d9\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"The + temperature in Paris is currently 21\xB0C.\"},\"sequence_number\":15}\n\nevent: + response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_0271f8b317d34f1a006a73982c49988191a842610ba590d1d9\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"The + temperature in Paris is currently 21\xB0C.\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":16}\n\nevent: + response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0271f8b317d34f1a006a73982b7d1c8191b3acc9be64742c71\",\"object\":\"response\",\"created_at\":1785960491,\"status\":\"completed\",\"background\":false,\"completed_at\":1785960492,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_0271f8b317d34f1a006a73982c49988191a842610ba590d1d9\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"The + temperature in Paris is currently 21\xB0C.\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Get + the current temperature for a city.\",\"name\":\"get_temperature\",\"output_schema\":null,\"parameters\":{\"properties\":{\"city\":{\"title\":\"City\",\"type\":\"string\"}},\"required\":[\"city\"],\"title\":\"get_temperature_Args\",\"type\":\"object\",\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":107,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":12,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":119},\"user\":null,\"metadata\":{}},\"sequence_number\":17}\n\n" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a2686eaf4d3a78a5-YYZ + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Wed, 05 Aug 2026 20:08:11 GMT + openai-processing-ms: + - '194' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-request-id: + - req_2889f3b1e8c04033872285f0543178cf + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/ai_sdk/cassettes/0.4.0/test_auto_ai_sdk.yaml b/py/src/braintrust/integrations/ai_sdk/cassettes/0.4.0/test_auto_ai_sdk.yaml new file mode 100644 index 00000000..4234d216 --- /dev/null +++ b/py/src/braintrust/integrations/ai_sdk/cassettes/0.4.0/test_auto_ai_sdk.yaml @@ -0,0 +1,127 @@ +interactions: +- request: + body: '{"input":[{"role":"user","content":[{"type":"input_text","text":"Reply + with the single word hello."}]}],"model":"gpt-4o-mini","stream":true}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '140' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 2.53.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.53.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: 'event: response.created + + data: {"type":"response.created","response":{"id":"resp_0de3548e807ca7c0006a73982dba3081a3973932182ec1509d","object":"response","created_at":1785960493,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + + event: response.in_progress + + data: {"type":"response.in_progress","response":{"id":"resp_0de3548e807ca7c0006a73982dba3081a3973932182ec1509d","object":"response","created_at":1785960493,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + + event: response.output_item.added + + data: {"type":"response.output_item.added","item":{"id":"msg_0de3548e807ca7c0006a73982e8d6881a3ba9daf0224d5861d","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":0,"sequence_number":2} + + + event: response.content_part.added + + data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_0de3548e807ca7c0006a73982e8d6881a3ba9daf0224d5861d","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"Hello","item_id":"msg_0de3548e807ca7c0006a73982e8d6881a3ba9daf0224d5861d","logprobs":[],"obfuscation":"RZ1ITY6toRN","output_index":0,"sequence_number":4} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":".","item_id":"msg_0de3548e807ca7c0006a73982e8d6881a3ba9daf0224d5861d","logprobs":[],"obfuscation":"LPSnZKrm6JfK2Zu","output_index":0,"sequence_number":5} + + + event: response.output_text.done + + data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_0de3548e807ca7c0006a73982e8d6881a3ba9daf0224d5861d","logprobs":[],"output_index":0,"sequence_number":6,"text":"Hello."} + + + event: response.content_part.done + + data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_0de3548e807ca7c0006a73982e8d6881a3ba9daf0224d5861d","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"Hello."},"sequence_number":7} + + + event: response.output_item.done + + data: {"type":"response.output_item.done","item":{"id":"msg_0de3548e807ca7c0006a73982e8d6881a3ba9daf0224d5861d","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Hello."}],"role":"assistant"},"output_index":0,"sequence_number":8} + + + event: response.completed + + data: {"type":"response.completed","response":{"id":"resp_0de3548e807ca7c0006a73982dba3081a3973932182ec1509d","object":"response","created_at":1785960493,"status":"completed","background":false,"completed_at":1785960494,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[{"id":"msg_0de3548e807ca7c0006a73982e8d6881a3ba9daf0224d5861d","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Hello."}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":14,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":0},"output_tokens":3,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":17},"user":null,"metadata":{}},"sequence_number":9} + + + ' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a2686ebd79520702-YYZ + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Wed, 05 Aug 2026 20:08:13 GMT + openai-processing-ms: + - '239' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-request-id: + - req_838f169b65ea40f2844fb4a4534c3547 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/ai_sdk/cassettes/0.4.0/test_provider_error_propagates_and_is_logged.yaml b/py/src/braintrust/integrations/ai_sdk/cassettes/0.4.0/test_provider_error_propagates_and_is_logged.yaml new file mode 100644 index 00000000..0dfb8ebe --- /dev/null +++ b/py/src/braintrust/integrations/ai_sdk/cassettes/0.4.0/test_provider_error_propagates_and_is_logged.yaml @@ -0,0 +1,78 @@ +interactions: +- request: + body: '{"input":[{"role":"user","content":[{"type":"input_text","text":"hello"}]}],"model":"braintrust-model-that-does-not-exist","stream":true}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '137' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 2.53.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.53.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "{\n \"error\": {\n \"message\": \"The requested model 'braintrust-model-that-does-not-exist' + does not exist.\",\n \"type\": \"invalid_request_error\",\n \"param\": + \"model\",\n \"code\": \"model_not_found\"\n }\n}" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a2686eb7199c051b-YYZ + connection: + - keep-alive + content-length: + - '202' + content-type: + - application/json + date: + - Wed, 05 Aug 2026 20:08:12 GMT + openai-processing-ms: + - '88' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-content-type-options: + - nosniff + x-request-id: + - req_1d663d61db2f472d868399fd538d67d2 + status: + code: 400 + message: Bad Request +version: 1 diff --git a/py/src/braintrust/integrations/ai_sdk/cassettes/0.4.0/test_stream_structured_multimodal_response.yaml b/py/src/braintrust/integrations/ai_sdk/cassettes/0.4.0/test_stream_structured_multimodal_response.yaml new file mode 100644 index 00000000..bc888547 --- /dev/null +++ b/py/src/braintrust/integrations/ai_sdk/cassettes/0.4.0/test_stream_structured_multimodal_response.yaml @@ -0,0 +1,142 @@ +interactions: +- request: + body: '{"input":[{"role":"system","content":"Return the requested value exactly."},{"role":"user","content":[{"type":"input_text","text":"What + is the dominant color of this image?"},{"type":"input_image","image_url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC"}]}],"model":"gpt-4o-mini","stream":true,"text":{"format":{"type":"json_schema","name":"ImageAnswer","schema":{"properties":{"color":{"title":"Color","type":"string"}},"required":["color"],"title":"ImageAnswer","type":"object","additionalProperties":false},"strict":true}}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '598' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 2.53.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.53.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: 'event: response.created + + data: {"type":"response.created","response":{"id":"resp_0443c63f8c2fc08f006a739828bacc819db4386764d789c0f4","object":"response","created_at":1785960488,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"json_schema","description":null,"name":"ImageAnswer","schema":{"properties":{"color":{"title":"Color","type":"string"}},"required":["color"],"title":"ImageAnswer","type":"object","additionalProperties":false},"strict":true},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + + event: response.in_progress + + data: {"type":"response.in_progress","response":{"id":"resp_0443c63f8c2fc08f006a739828bacc819db4386764d789c0f4","object":"response","created_at":1785960488,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"json_schema","description":null,"name":"ImageAnswer","schema":{"properties":{"color":{"title":"Color","type":"string"}},"required":["color"],"title":"ImageAnswer","type":"object","additionalProperties":false},"strict":true},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + + event: response.output_item.added + + data: {"type":"response.output_item.added","item":{"id":"msg_0443c63f8c2fc08f006a73982a0d64819d8baa658b6dcd3dd7","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":0,"sequence_number":2} + + + event: response.content_part.added + + data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_0443c63f8c2fc08f006a73982a0d64819d8baa658b6dcd3dd7","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"{\"","item_id":"msg_0443c63f8c2fc08f006a73982a0d64819d8baa658b6dcd3dd7","logprobs":[],"obfuscation":"0n76qfdcADztW7","output_index":0,"sequence_number":4} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"color","item_id":"msg_0443c63f8c2fc08f006a73982a0d64819d8baa658b6dcd3dd7","logprobs":[],"obfuscation":"Md7y5Uzog2D","output_index":0,"sequence_number":5} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"\":\"","item_id":"msg_0443c63f8c2fc08f006a73982a0d64819d8baa658b6dcd3dd7","logprobs":[],"obfuscation":"NeN6atzephU24","output_index":0,"sequence_number":6} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"red","item_id":"msg_0443c63f8c2fc08f006a73982a0d64819d8baa658b6dcd3dd7","logprobs":[],"obfuscation":"coH7AsYFC5drN","output_index":0,"sequence_number":7} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"\"}","item_id":"msg_0443c63f8c2fc08f006a73982a0d64819d8baa658b6dcd3dd7","logprobs":[],"obfuscation":"kO1qFMaQh2fI5e","output_index":0,"sequence_number":8} + + + event: response.output_text.done + + data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_0443c63f8c2fc08f006a73982a0d64819d8baa658b6dcd3dd7","logprobs":[],"output_index":0,"sequence_number":9,"text":"{\"color\":\"red\"}"} + + + event: response.content_part.done + + data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_0443c63f8c2fc08f006a73982a0d64819d8baa658b6dcd3dd7","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"{\"color\":\"red\"}"},"sequence_number":10} + + + event: response.output_item.done + + data: {"type":"response.output_item.done","item":{"id":"msg_0443c63f8c2fc08f006a73982a0d64819d8baa658b6dcd3dd7","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"{\"color\":\"red\"}"}],"role":"assistant"},"output_index":0,"sequence_number":11} + + + event: response.completed + + data: {"type":"response.completed","response":{"id":"resp_0443c63f8c2fc08f006a739828bacc819db4386764d789c0f4","object":"response","created_at":1785960488,"status":"completed","background":false,"completed_at":1785960490,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[{"id":"msg_0443c63f8c2fc08f006a73982a0d64819d8baa658b6dcd3dd7","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"{\"color\":\"red\"}"}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"json_schema","description":null,"name":"ImageAnswer","schema":{"properties":{"color":{"title":"Color","type":"string"}},"required":["color"],"title":"ImageAnswer","type":"object","additionalProperties":false},"strict":true},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":8557,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":0},"output_tokens":6,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":8563},"user":null,"metadata":{}},"sequence_number":12} + + + ' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a2686e9d1b4d9113-YYZ + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Wed, 05 Aug 2026 20:08:08 GMT + openai-processing-ms: + - '265' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-request-id: + - req_bc18a8f88897486f91eae08f4e66ddc8 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/ai_sdk/cassettes/latest/test_agent_tool_loop_emits_run_model_and_tool_hierarchy.yaml b/py/src/braintrust/integrations/ai_sdk/cassettes/latest/test_agent_tool_loop_emits_run_model_and_tool_hierarchy.yaml new file mode 100644 index 00000000..6996c2d7 --- /dev/null +++ b/py/src/braintrust/integrations/ai_sdk/cassettes/latest/test_agent_tool_loop_emits_run_model_and_tool_hierarchy.yaml @@ -0,0 +1,242 @@ +interactions: +- request: + body: '{"input":[{"role":"system","content":"Always use get_temperature once before + answering. Be concise."},{"role":"user","content":[{"type":"input_text","text":"What + is the temperature in Paris?"}]}],"model":"gpt-4o-mini","stream":true,"tools":[{"type":"function","name":"get_temperature","description":"Get + the current temperature for a city.","parameters":{"properties":{"city":{"title":"City","type":"string"}},"required":["city"],"title":"get_temperature_Args","type":"object"}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '480' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 2.53.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.53.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: 'event: response.created + + data: {"type":"response.created","response":{"id":"resp_08080560c4acb3ec006a7396bf6b1c81a19f4a492a8b284089","object":"response","created_at":1785960127,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[{"type":"function","description":"Get + the current temperature for a city.","name":"get_temperature","output_schema":null,"parameters":{"properties":{"city":{"title":"City","type":"string"}},"required":["city"],"title":"get_temperature_Args","type":"object","additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + + event: response.in_progress + + data: {"type":"response.in_progress","response":{"id":"resp_08080560c4acb3ec006a7396bf6b1c81a19f4a492a8b284089","object":"response","created_at":1785960127,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[{"type":"function","description":"Get + the current temperature for a city.","name":"get_temperature","output_schema":null,"parameters":{"properties":{"city":{"title":"City","type":"string"}},"required":["city"],"title":"get_temperature_Args","type":"object","additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + + event: response.output_item.added + + data: {"type":"response.output_item.added","item":{"id":"fc_08080560c4acb3ec006a7396c07e9881a18069857a4121e256","type":"function_call","status":"in_progress","arguments":"","call_id":"call_rp1SkXbY2831eTnTuGu6A5XN","name":"get_temperature"},"output_index":0,"sequence_number":2} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"{\"","item_id":"fc_08080560c4acb3ec006a7396c07e9881a18069857a4121e256","obfuscation":"8rDCOEQnkJSuKy","output_index":0,"sequence_number":3} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"city","item_id":"fc_08080560c4acb3ec006a7396c07e9881a18069857a4121e256","obfuscation":"L2ohNVdaECRK","output_index":0,"sequence_number":4} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"\":\"","item_id":"fc_08080560c4acb3ec006a7396c07e9881a18069857a4121e256","obfuscation":"xheZvbFAZknDU","output_index":0,"sequence_number":5} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"Paris","item_id":"fc_08080560c4acb3ec006a7396c07e9881a18069857a4121e256","obfuscation":"TAMEMKfoVIS","output_index":0,"sequence_number":6} + + + event: response.function_call_arguments.delta + + data: {"type":"response.function_call_arguments.delta","delta":"\"}","item_id":"fc_08080560c4acb3ec006a7396c07e9881a18069857a4121e256","obfuscation":"TGx6gMf2nPDCxX","output_index":0,"sequence_number":7} + + + event: response.function_call_arguments.done + + data: {"type":"response.function_call_arguments.done","arguments":"{\"city\":\"Paris\"}","item_id":"fc_08080560c4acb3ec006a7396c07e9881a18069857a4121e256","output_index":0,"sequence_number":8} + + + event: response.output_item.done + + data: {"type":"response.output_item.done","item":{"id":"fc_08080560c4acb3ec006a7396c07e9881a18069857a4121e256","type":"function_call","status":"completed","arguments":"{\"city\":\"Paris\"}","call_id":"call_rp1SkXbY2831eTnTuGu6A5XN","name":"get_temperature"},"output_index":0,"sequence_number":9} + + + event: response.completed + + data: {"type":"response.completed","response":{"id":"resp_08080560c4acb3ec006a7396bf6b1c81a19f4a492a8b284089","object":"response","created_at":1785960127,"status":"completed","background":false,"completed_at":1785960128,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[{"id":"fc_08080560c4acb3ec006a7396c07e9881a18069857a4121e256","type":"function_call","status":"completed","arguments":"{\"city\":\"Paris\"}","call_id":"call_rp1SkXbY2831eTnTuGu6A5XN","name":"get_temperature"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[{"type":"function","description":"Get + the current temperature for a city.","name":"get_temperature","output_schema":null,"parameters":{"properties":{"city":{"title":"City","type":"string"}},"required":["city"],"title":"get_temperature_Args","type":"object","additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":70,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":0},"output_tokens":15,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":85},"user":null,"metadata":{}},"sequence_number":10} + + + ' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a26865cbac26ac66-YYZ + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Wed, 05 Aug 2026 20:02:07 GMT + openai-processing-ms: + - '185' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-request-id: + - req_6ed676de01004cb1a4c0438cb4ee4519 + status: + code: 200 + message: OK +- request: + body: '{"input":[{"role":"system","content":"Always use get_temperature once before + answering. Be concise."},{"role":"user","content":[{"type":"input_text","text":"What + is the temperature in Paris?"}]},{"type":"item_reference","id":"fc_08080560c4acb3ec006a7396c07e9881a18069857a4121e256"},{"type":"function_call_output","call_id":"call_rp1SkXbY2831eTnTuGu6A5XN","output":"{\"city\":\"Paris\",\"temperature\":21,\"unit\":\"celsius\"}"}],"model":"gpt-4o-mini","stream":true,"tools":[{"type":"function","name":"get_temperature","description":"Get + the current temperature for a city.","parameters":{"properties":{"city":{"title":"City","type":"string"}},"required":["city"],"title":"get_temperature_Args","type":"object"}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '713' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 2.53.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.53.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_08080560c4acb3ec006a7396c0eb6c81a1aef415569140d781\",\"object\":\"response\",\"created_at\":1785960128,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Get + the current temperature for a city.\",\"name\":\"get_temperature\",\"output_schema\":null,\"parameters\":{\"properties\":{\"city\":{\"title\":\"City\",\"type\":\"string\"}},\"required\":[\"city\"],\"title\":\"get_temperature_Args\",\"type\":\"object\",\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}\n\nevent: + response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_08080560c4acb3ec006a7396c0eb6c81a1aef415569140d781\",\"object\":\"response\",\"created_at\":1785960128,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Get + the current temperature for a city.\",\"name\":\"get_temperature\",\"output_schema\":null,\"parameters\":{\"properties\":{\"city\":{\"title\":\"City\",\"type\":\"string\"}},\"required\":[\"city\"],\"title\":\"get_temperature_Args\",\"type\":\"object\",\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}\n\nevent: + response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_08080560c4acb3ec006a7396c1867881a1ad8381cdf1c921ee\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}\n\nevent: + response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_08080560c4acb3ec006a7396c1867881a1ad8381cdf1c921ee\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"The\",\"item_id\":\"msg_08080560c4acb3ec006a7396c1867881a1ad8381cdf1c921ee\",\"logprobs\":[],\"obfuscation\":\"XAvzEfzWPTSOM\",\"output_index\":0,\"sequence_number\":4}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + temperature\",\"item_id\":\"msg_08080560c4acb3ec006a7396c1867881a1ad8381cdf1c921ee\",\"logprobs\":[],\"obfuscation\":\"XO9t\",\"output_index\":0,\"sequence_number\":5}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + in\",\"item_id\":\"msg_08080560c4acb3ec006a7396c1867881a1ad8381cdf1c921ee\",\"logprobs\":[],\"obfuscation\":\"7h28RBchN2DNu\",\"output_index\":0,\"sequence_number\":6}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + Paris\",\"item_id\":\"msg_08080560c4acb3ec006a7396c1867881a1ad8381cdf1c921ee\",\"logprobs\":[],\"obfuscation\":\"fP3o7TORXv\",\"output_index\":0,\"sequence_number\":7}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + is\",\"item_id\":\"msg_08080560c4acb3ec006a7396c1867881a1ad8381cdf1c921ee\",\"logprobs\":[],\"obfuscation\":\"fL1oUzJhI6erx\",\"output_index\":0,\"sequence_number\":8}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + \",\"item_id\":\"msg_08080560c4acb3ec006a7396c1867881a1ad8381cdf1c921ee\",\"logprobs\":[],\"obfuscation\":\"bH6idU9licqibot\",\"output_index\":0,\"sequence_number\":9}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"21\",\"item_id\":\"msg_08080560c4acb3ec006a7396c1867881a1ad8381cdf1c921ee\",\"logprobs\":[],\"obfuscation\":\"PV4eKbbfU6YET7\",\"output_index\":0,\"sequence_number\":10}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\xB0C\",\"item_id\":\"msg_08080560c4acb3ec006a7396c1867881a1ad8381cdf1c921ee\",\"logprobs\":[],\"obfuscation\":\"GXUzvDINdJD2hT\",\"output_index\":0,\"sequence_number\":11}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\".\",\"item_id\":\"msg_08080560c4acb3ec006a7396c1867881a1ad8381cdf1c921ee\",\"logprobs\":[],\"obfuscation\":\"5Q8wqBxiALiYk32\",\"output_index\":0,\"sequence_number\":12}\n\nevent: + response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_08080560c4acb3ec006a7396c1867881a1ad8381cdf1c921ee\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":13,\"text\":\"The + temperature in Paris is 21\xB0C.\"}\n\nevent: response.content_part.done\ndata: + {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_08080560c4acb3ec006a7396c1867881a1ad8381cdf1c921ee\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"The + temperature in Paris is 21\xB0C.\"},\"sequence_number\":14}\n\nevent: response.output_item.done\ndata: + {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_08080560c4acb3ec006a7396c1867881a1ad8381cdf1c921ee\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"The + temperature in Paris is 21\xB0C.\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":15}\n\nevent: + response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_08080560c4acb3ec006a7396c0eb6c81a1aef415569140d781\",\"object\":\"response\",\"created_at\":1785960128,\"status\":\"completed\",\"background\":false,\"completed_at\":1785960129,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_08080560c4acb3ec006a7396c1867881a1ad8381cdf1c921ee\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"The + temperature in Paris is 21\xB0C.\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Get + the current temperature for a city.\",\"name\":\"get_temperature\",\"output_schema\":null,\"parameters\":{\"properties\":{\"city\":{\"title\":\"City\",\"type\":\"string\"}},\"required\":[\"city\"],\"title\":\"get_temperature_Args\",\"type\":\"object\",\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":107,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":11,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":118},\"user\":null,\"metadata\":{}},\"sequence_number\":16}\n\n" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a26865d56d60ac66-YYZ + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Wed, 05 Aug 2026 20:02:09 GMT + openai-processing-ms: + - '182' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-request-id: + - req_d6511055c761461bac794ec1c91af53c + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/ai_sdk/cassettes/latest/test_auto_ai_sdk.yaml b/py/src/braintrust/integrations/ai_sdk/cassettes/latest/test_auto_ai_sdk.yaml new file mode 100644 index 00000000..b6de473e --- /dev/null +++ b/py/src/braintrust/integrations/ai_sdk/cassettes/latest/test_auto_ai_sdk.yaml @@ -0,0 +1,127 @@ +interactions: +- request: + body: '{"input":[{"role":"user","content":[{"type":"input_text","text":"Reply + with the single word hello."}]}],"model":"gpt-4o-mini","stream":true}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '140' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 2.53.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.53.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: 'event: response.created + + data: {"type":"response.created","response":{"id":"resp_03ca9182fb54194f006a7396d4649881a3b8e2c58f3183be99","object":"response","created_at":1785960148,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + + event: response.in_progress + + data: {"type":"response.in_progress","response":{"id":"resp_03ca9182fb54194f006a7396d4649881a3b8e2c58f3183be99","object":"response","created_at":1785960148,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + + event: response.output_item.added + + data: {"type":"response.output_item.added","item":{"id":"msg_03ca9182fb54194f006a7396d4d3c481a394ddb70909aa6257","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":0,"sequence_number":2} + + + event: response.content_part.added + + data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_03ca9182fb54194f006a7396d4d3c481a394ddb70909aa6257","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"Hello","item_id":"msg_03ca9182fb54194f006a7396d4d3c481a394ddb70909aa6257","logprobs":[],"obfuscation":"r7t9hiWR8GJ","output_index":0,"sequence_number":4} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":".","item_id":"msg_03ca9182fb54194f006a7396d4d3c481a394ddb70909aa6257","logprobs":[],"obfuscation":"XX9XY5xA7H9cJwJ","output_index":0,"sequence_number":5} + + + event: response.output_text.done + + data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_03ca9182fb54194f006a7396d4d3c481a394ddb70909aa6257","logprobs":[],"output_index":0,"sequence_number":6,"text":"Hello."} + + + event: response.content_part.done + + data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_03ca9182fb54194f006a7396d4d3c481a394ddb70909aa6257","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"Hello."},"sequence_number":7} + + + event: response.output_item.done + + data: {"type":"response.output_item.done","item":{"id":"msg_03ca9182fb54194f006a7396d4d3c481a394ddb70909aa6257","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Hello."}],"role":"assistant"},"output_index":0,"sequence_number":8} + + + event: response.completed + + data: {"type":"response.completed","response":{"id":"resp_03ca9182fb54194f006a7396d4649881a3b8e2c58f3183be99","object":"response","created_at":1785960148,"status":"completed","background":false,"completed_at":1785960148,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[{"id":"msg_03ca9182fb54194f006a7396d4d3c481a394ddb70909aa6257","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Hello."}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":14,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":0},"output_tokens":3,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":17},"user":null,"metadata":{}},"sequence_number":9} + + + ' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a268664efb4b36d8-YYZ + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Wed, 05 Aug 2026 20:02:28 GMT + openai-processing-ms: + - '197' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-request-id: + - req_58839833189e49e08373c7230ff7fe29 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/ai_sdk/cassettes/latest/test_provider_error_propagates_and_is_logged.yaml b/py/src/braintrust/integrations/ai_sdk/cassettes/latest/test_provider_error_propagates_and_is_logged.yaml new file mode 100644 index 00000000..9db6ecda --- /dev/null +++ b/py/src/braintrust/integrations/ai_sdk/cassettes/latest/test_provider_error_propagates_and_is_logged.yaml @@ -0,0 +1,78 @@ +interactions: +- request: + body: '{"input":[{"role":"user","content":[{"type":"input_text","text":"hello"}]}],"model":"braintrust-model-that-does-not-exist","stream":true}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '137' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 2.53.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.53.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "{\n \"error\": {\n \"message\": \"The requested model 'braintrust-model-that-does-not-exist' + does not exist.\",\n \"type\": \"invalid_request_error\",\n \"param\": + \"model\",\n \"code\": \"model_not_found\"\n }\n}" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a2686612fd24cca0-YYZ + connection: + - keep-alive + content-length: + - '202' + content-type: + - application/json + date: + - Wed, 05 Aug 2026 20:02:18 GMT + openai-processing-ms: + - '92' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-content-type-options: + - nosniff + x-request-id: + - req_9f2deacd3ec5456dba6655b4c6250a82 + status: + code: 400 + message: Bad Request +version: 1 diff --git a/py/src/braintrust/integrations/ai_sdk/cassettes/latest/test_stream_structured_multimodal_response.yaml b/py/src/braintrust/integrations/ai_sdk/cassettes/latest/test_stream_structured_multimodal_response.yaml new file mode 100644 index 00000000..d954a93c --- /dev/null +++ b/py/src/braintrust/integrations/ai_sdk/cassettes/latest/test_stream_structured_multimodal_response.yaml @@ -0,0 +1,142 @@ +interactions: +- request: + body: '{"input":[{"role":"system","content":"Return the requested value exactly."},{"role":"user","content":[{"type":"input_text","text":"What + is the dominant color of this image?"},{"type":"input_image","image_url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC"}]}],"model":"gpt-4o-mini","stream":true,"text":{"format":{"type":"json_schema","name":"ImageAnswer","schema":{"properties":{"color":{"title":"Color","type":"string"}},"required":["color"],"title":"ImageAnswer","type":"object","additionalProperties":false},"strict":true}}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '598' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 2.53.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.53.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: 'event: response.created + + data: {"type":"response.created","response":{"id":"resp_0d9fb1b185308b6e006a7396b4b03081a2a91c5b052247cbb7","object":"response","created_at":1785960116,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"json_schema","description":null,"name":"ImageAnswer","schema":{"properties":{"color":{"title":"Color","type":"string"}},"required":["color"],"title":"ImageAnswer","type":"object","additionalProperties":false},"strict":true},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + + event: response.in_progress + + data: {"type":"response.in_progress","response":{"id":"resp_0d9fb1b185308b6e006a7396b4b03081a2a91c5b052247cbb7","object":"response","created_at":1785960116,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"json_schema","description":null,"name":"ImageAnswer","schema":{"properties":{"color":{"title":"Color","type":"string"}},"required":["color"],"title":"ImageAnswer","type":"object","additionalProperties":false},"strict":true},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + + event: response.output_item.added + + data: {"type":"response.output_item.added","item":{"id":"msg_0d9fb1b185308b6e006a7396b62e2081a2b06aec49121870bb","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":0,"sequence_number":2} + + + event: response.content_part.added + + data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_0d9fb1b185308b6e006a7396b62e2081a2b06aec49121870bb","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"{\"","item_id":"msg_0d9fb1b185308b6e006a7396b62e2081a2b06aec49121870bb","logprobs":[],"obfuscation":"730xuwZ8NYapAr","output_index":0,"sequence_number":4} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"color","item_id":"msg_0d9fb1b185308b6e006a7396b62e2081a2b06aec49121870bb","logprobs":[],"obfuscation":"YDC4eqowb5S","output_index":0,"sequence_number":5} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"\":\"","item_id":"msg_0d9fb1b185308b6e006a7396b62e2081a2b06aec49121870bb","logprobs":[],"obfuscation":"7NToVtzHCquDD","output_index":0,"sequence_number":6} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"red","item_id":"msg_0d9fb1b185308b6e006a7396b62e2081a2b06aec49121870bb","logprobs":[],"obfuscation":"GjiwKLaqd0kDv","output_index":0,"sequence_number":7} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"\"}","item_id":"msg_0d9fb1b185308b6e006a7396b62e2081a2b06aec49121870bb","logprobs":[],"obfuscation":"0pgGs67zmndDXP","output_index":0,"sequence_number":8} + + + event: response.output_text.done + + data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_0d9fb1b185308b6e006a7396b62e2081a2b06aec49121870bb","logprobs":[],"output_index":0,"sequence_number":9,"text":"{\"color\":\"red\"}"} + + + event: response.content_part.done + + data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_0d9fb1b185308b6e006a7396b62e2081a2b06aec49121870bb","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"{\"color\":\"red\"}"},"sequence_number":10} + + + event: response.output_item.done + + data: {"type":"response.output_item.done","item":{"id":"msg_0d9fb1b185308b6e006a7396b62e2081a2b06aec49121870bb","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"{\"color\":\"red\"}"}],"role":"assistant"},"output_index":0,"sequence_number":11} + + + event: response.completed + + data: {"type":"response.completed","response":{"id":"resp_0d9fb1b185308b6e006a7396b4b03081a2a91c5b052247cbb7","object":"response","created_at":1785960116,"status":"completed","background":false,"completed_at":1785960118,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[{"id":"msg_0d9fb1b185308b6e006a7396b62e2081a2b06aec49121870bb","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"{\"color\":\"red\"}"}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"json_schema","description":null,"name":"ImageAnswer","schema":{"properties":{"color":{"title":"Color","type":"string"}},"required":["color"],"title":"ImageAnswer","type":"object","additionalProperties":false},"strict":true},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":8557,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":0},"output_tokens":6,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":8563},"user":null,"metadata":{}},"sequence_number":12} + + + ' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a2686588bbb2f41f-YYZ + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Wed, 05 Aug 2026 20:01:57 GMT + openai-processing-ms: + - '376' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-request-id: + - req_2e71a80eaf564cf48904773c180549e7 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/ai_sdk/integration.py b/py/src/braintrust/integrations/ai_sdk/integration.py new file mode 100644 index 00000000..64dbc192 --- /dev/null +++ b/py/src/braintrust/integrations/ai_sdk/integration.py @@ -0,0 +1,15 @@ +"""Vercel AI SDK for Python integration orchestration.""" + +from braintrust.integrations.base import BaseIntegration + +from .patchers import TelemetryAdapterPatcher + + +class AISDKIntegration(BaseIntegration): + """Braintrust instrumentation for Vercel AI SDK for Python.""" + + name = "ai_sdk" + import_names = ("ai",) + distribution_names = ("ai",) + min_version = "0.4.0" + patchers = (TelemetryAdapterPatcher,) diff --git a/py/src/braintrust/integrations/ai_sdk/patchers.py b/py/src/braintrust/integrations/ai_sdk/patchers.py new file mode 100644 index 00000000..a1ce8f24 --- /dev/null +++ b/py/src/braintrust/integrations/ai_sdk/patchers.py @@ -0,0 +1,53 @@ +"""Registration patcher for Vercel AI SDK telemetry.""" + +from typing import Any, ClassVar + +from braintrust.integrations.base import CallbackPatcher + +from .tracing import BraintrustAISDKAdapter + + +_ADAPTER: BraintrustAISDKAdapter | None = None + + +def _adapter_registered() -> bool: + return _ADAPTER is not None + + +def _register_adapter() -> bool: + global _ADAPTER # noqa: PLW0603 + if _ADAPTER is not None: + return True + + from ai import experimental_telemetry # pylint: disable=import-error + + adapter = BraintrustAISDKAdapter() + experimental_telemetry.register(adapter) + _ADAPTER = adapter + return True + + +def unregister_adapter() -> bool: + """Unregister the process-wide Braintrust telemetry adapter.""" + global _ADAPTER # noqa: PLW0603 + if _ADAPTER is None: + return True + + from ai import experimental_telemetry # pylint: disable=import-error + + adapter = _ADAPTER + try: + experimental_telemetry.unregister(adapter) + except ValueError: + pass + _ADAPTER = None + return True + + +class TelemetryAdapterPatcher(CallbackPatcher): + """Register one Braintrust adapter with ``ai.experimental_telemetry``.""" + + name: ClassVar[str] = "ai.experimental_telemetry.adapter" + target_module: ClassVar[str] = "ai.experimental_telemetry" + callback: ClassVar[Any] = staticmethod(_register_adapter) + state_getter: ClassVar[Any] = staticmethod(_adapter_registered) diff --git a/py/src/braintrust/integrations/ai_sdk/test_ai_sdk.py b/py/src/braintrust/integrations/ai_sdk/test_ai_sdk.py new file mode 100644 index 00000000..f2b31c8c --- /dev/null +++ b/py/src/braintrust/integrations/ai_sdk/test_ai_sdk.py @@ -0,0 +1,257 @@ +# pylint: disable=import-error,no-member +# pyright: reportUntypedFunctionDecorator=false +# pyright: reportUnknownMemberType=false +# pyright: reportUnknownVariableType=false +import base64 + +import ai +import pytest +from braintrust import Attachment, logger, setup_ai_sdk +from braintrust.integrations.ai_sdk import patch_ai_sdk, unpatch_ai_sdk +from braintrust.integrations.ai_sdk.tracing import _shape_input_messages +from braintrust.integrations.test_utils import verify_autoinstrument_script +from braintrust.span_types import SpanTypeAttribute +from braintrust.test_helpers import init_test_logger +from pydantic import BaseModel + + +PROJECT_NAME = "test-ai-sdk" +MODEL = "openai:gpt-4o-mini" +_RED_PIXEL_PNG = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC" +) + + +class ImageAnswer(BaseModel): + color: str + + +@pytest.fixture(scope="module", autouse=True) +def setup_integration(): + assert setup_ai_sdk(project_name=PROJECT_NAME) + + +@pytest.fixture +def memory_logger(): + init_test_logger(PROJECT_NAME) + with logger._internal_with_memory_background_logger() as background_logger: + background_logger.pop() + yield background_logger + + +def _span_named(spans, name): + return next(span for span in spans if span["span_attributes"]["name"] == name) + + +def _contains_attachment(value): + if isinstance(value, Attachment): + return True + if isinstance(value, dict): + return any(_contains_attachment(item) for item in value.values()) + if isinstance(value, list): + return any(_contains_attachment(item) for item in value) + return False + + +def _assert_ai_sdk_origin(span): + assert span["context"]["span_origin"]["instrumentation"]["name"] == "ai-sdk-auto" + + +@pytest.mark.vcr +@pytest.mark.asyncio +async def test_stream_structured_multimodal_response(memory_logger): + model = ai.get_model(MODEL) + messages = [ + ai.system_message("Return the requested value exactly."), + ai.user_message( + "What is the dominant color of this image?", + ai.file_part(_RED_PIXEL_PNG, media_type="image/png", filename="red.png"), + ), + ] + + chunks = [] + async with ai.stream(model, messages, output_type=ImageAnswer) as stream: + async for event in stream: + if isinstance(event, ai.events.TextDelta): + chunks.append(event.chunk) + + assert chunks + assert stream.output.color.lower() == "red" + + spans = memory_logger.pop() + assert len(spans) == 1 + span = _span_named(spans, "ai.stream") + assert span["span_attributes"]["type"] == SpanTypeAttribute.TASK + assert _contains_attachment(span["input"]) + assert span["output"][0]["message"]["role"] == "assistant" + assert "red" in span["output"][0]["message"]["content"].lower() + assert span["metadata"]["provider"] == "openai" + assert "gpt-4o-mini" in span["metadata"]["model"] + assert span["metadata"]["response_id"] + assert span["metadata"]["output_type"] == "ImageAnswer" + for token_metric in ("tokens", "prompt_tokens", "completion_tokens"): + assert token_metric not in span["metrics"] + assert span["metrics"]["time_to_first_token"] >= 0 + _assert_ai_sdk_origin(span) + + +@pytest.mark.vcr +@pytest.mark.asyncio +async def test_agent_tool_loop_emits_run_model_and_tool_hierarchy(memory_logger): + @ai.tool + async def get_temperature(city: str) -> dict[str, object]: + """Get the current temperature for a city.""" + return {"city": city, "temperature": 21, "unit": "celsius"} + + model = ai.get_model(MODEL) + agent = ai.Agent(tools=[get_temperature]) + messages = [ + ai.system_message("Always use get_temperature once before answering. Be concise."), + ai.user_message("What is the temperature in Paris?"), + ] + + text_chunks = [] + async with agent.run(model, messages) as stream: + async for event in stream: + if isinstance(event, ai.events.TextDelta): + text_chunks.append(event.chunk) + + assert "21" in "".join(text_chunks) + + spans = memory_logger.pop() + run_span = _span_named(spans, "Agent.run") + stream_spans = [span for span in spans if span["span_attributes"]["name"] == "ai.stream"] + tool_span = _span_named(spans, "get_temperature") + turn_spans = [span for span in spans if span["span_attributes"]["name"] == "Loop Turn"] + + assert run_span["span_attributes"]["type"] == SpanTypeAttribute.TASK + assert len(stream_spans) == 2 + assert not [span for span in spans if span["span_attributes"].get("type") == SpanTypeAttribute.LLM] + assert len(turn_spans) == 2 + assert tool_span["span_attributes"]["type"] == SpanTypeAttribute.TOOL + assert tool_span["input"]["city"].lower() == "paris" + assert tool_span["output"] == {"city": "Paris", "temperature": 21, "unit": "celsius"} + assert tool_span["metadata"]["tool_call_id"] + + span_by_id = {span["span_id"]: span for span in spans} + for turn_span in turn_spans: + assert turn_span["span_parents"] == [run_span["span_id"]] + for child_span in [*stream_spans, tool_span]: + (parent_id,) = child_span["span_parents"] + assert span_by_id[parent_id]["span_attributes"]["name"] == "Loop Turn" + + for stream_span in stream_spans: + assert stream_span["span_attributes"]["type"] == SpanTypeAttribute.TASK + assert stream_span["metadata"]["provider"] == "openai" + assert stream_span["metadata"]["tools"] == [{"type": "function", "function": {"name": "get_temperature"}}] + for token_metric in ("tokens", "prompt_tokens", "completion_tokens"): + assert token_metric not in stream_span["metrics"] + _assert_ai_sdk_origin(stream_span) + + tool_call_stream = next(span for span in stream_spans if span["output"][0]["finish_reason"] == "tool_calls") + tool_call = tool_call_stream["output"][0]["message"]["tool_calls"][0] + assert tool_call["function"]["name"] == "get_temperature" + assert isinstance(tool_call["function"]["arguments"], str) + followup_stream = next(span for span in stream_spans if span is not tool_call_stream) + tool_result_message = next(message for message in followup_stream["input"] if message["role"] == "tool") + assert tool_result_message["tool_call_id"] == tool_call["id"] + assert isinstance(tool_result_message["content"], str) + + for token_metric in ("tokens", "prompt_tokens", "completion_tokens"): + assert token_metric not in run_span.get("metrics", {}) + assert "21" in str(run_span["output"]) + for span in spans: + _assert_ai_sdk_origin(span) + + +def test_tool_result_input_uses_model_facing_value(): + result = ai.types.messages.ToolResultPart( + tool_call_id="call_1", + tool_name="search", + result={"documents": ["full document contents"]}, + ) + result.set_model_input({"summary": "compact model context"}) + message = ai.types.messages.Message(role="tool", parts=[result]) + + expected = '{"summary":"compact model context"}' + assert _shape_input_messages([message])[0]["content"] == expected + assert _shape_input_messages([message.model_dump(mode="python")])[0]["content"] == expected + + +@pytest.mark.asyncio +async def test_durable_replay_preserves_parent_hierarchy(memory_logger): + sink = ai.experimental_telemetry.DictSink() + async with ai.experimental_telemetry.use_sink(sink): + async with ai.experimental_telemetry.span( + ai.experimental_telemetry.RunSpanData( + agent="DurableAgent", + model="gpt-4o-mini", + messages=[ai.user_message("hello")], + provider="openai", + ) + ): + async with ai.experimental_telemetry.span(ai.experimental_telemetry.LoopTurnSpanData()): + pass + + payload = [span.model_dump(mode="json") for span in sink.finished_spans] + await ai.experimental_telemetry.push_all(payload) + + spans = memory_logger.pop() + run_span = _span_named(spans, "Agent.run") + turn_span = _span_named(spans, "Loop Turn") + assert turn_span["span_parents"] == [run_span["span_id"]] + assert turn_span["root_span_id"] == run_span["root_span_id"] + + +@pytest.mark.asyncio +async def test_custom_and_hook_spans_preserve_attributes(memory_logger): + async with ai.experimental_telemetry.span("retrieve_context") as custom_span: + custom_span.set_attrs(query="Paris", count=2) + + hook_data = ai.experimental_telemetry.HookSpanData( + label="approve_call_1", + hook_type="approval", + metadata={"tool": "send_email"}, + tool_call_id="call_1", + ) + async with ai.experimental_telemetry.span(hook_data): + hook_data.status = "resolved" + hook_data.resolution = {"granted": True} + + spans = memory_logger.pop() + custom = _span_named(spans, "retrieve_context") + hook = _span_named(spans, "Hook: approve_call_1") + assert custom["metadata"] == {"query": "Paris", "count": 2} + assert hook["span_attributes"]["type"] == SpanTypeAttribute.TASK + assert hook["metadata"]["tool_call_id"] == "call_1" + assert hook["output"] == {"status": "resolved", "resolution": {"granted": True}} + + +@pytest.mark.vcr +@pytest.mark.asyncio +async def test_provider_error_propagates_and_is_logged(memory_logger): + model = ai.get_model("openai:braintrust-model-that-does-not-exist") + + with pytest.raises(Exception) as exc_info: # provider exception type is intentionally preserved + async with ai.stream(model, [ai.user_message("hello")]) as stream: + async for _ in stream: + pass + + assert "model" in str(exc_info.value).lower() + spans = memory_logger.pop() + assert len(spans) == 1 + span = _span_named(spans, "ai.stream") + assert span["error"] + assert "model" in span["error"].lower() + _assert_ai_sdk_origin(span) + + +def test_registration_and_unregistration_are_idempotent(): + assert unpatch_ai_sdk() + assert unpatch_ai_sdk() + assert patch_ai_sdk() + assert patch_ai_sdk() + + +def test_auto_instrument_ai_sdk_subprocess(): + verify_autoinstrument_script("test_auto_ai_sdk.py") diff --git a/py/src/braintrust/integrations/ai_sdk/tracing.py b/py/src/braintrust/integrations/ai_sdk/tracing.py new file mode 100644 index 00000000..84aef833 --- /dev/null +++ b/py/src/braintrust/integrations/ai_sdk/tracing.py @@ -0,0 +1,516 @@ +"""Translate Vercel AI SDK telemetry spans into Braintrust spans.""" + +import json +from collections import OrderedDict +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from braintrust.integrations.utils import ( + _is_supported_metric_value, + _materialize_attachment, + _parse_openai_usage_metrics, +) +from braintrust.logger import Span +from braintrust.logger import start_span as _bt_start_span +from braintrust.span_types import SpanTypeAttribute +from braintrust.util import clean_nones + + +if TYPE_CHECKING: + from ai.types.messages import ( # pylint: disable=import-error + BuiltinToolCallPart, + BuiltinToolReturnPart, + FilePart, + Part, + ToolCallPart, + ToolResultPart, + ) + + +_INSTRUMENTATION = "ai-sdk-auto" +# Parents are only consulted by children that start after the parent ended +# (durable replay, out-of-order pushes), so this holds finished spans just +# long enough for one replayed batch. +_MAX_ARCHIVED_PARENTS = 1_000 +_CONTENT_PART_KINDS = frozenset({"text", "reasoning", "file", "hook"}) +_TOOL_CALL_KINDS = frozenset({"tool_call", "builtin_tool_call"}) +_TOOL_RESULT_KINDS = frozenset({"tool_result", "builtin_tool_return"}) +_USAGE_NAME_MAP = { + "input_tokens": "prompt_tokens", + "output_tokens": "completion_tokens", + "total_tokens": "tokens", + "reasoning_tokens": "completion_reasoning_tokens", + "cache_read_tokens": "prompt_cached_tokens", + "cache_write_tokens": "prompt_cache_creation_tokens", +} +_USAGE_PREFIX_MAP = {"input": "prompt", "output": "completion"} + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + +def _value(obj: Any, name: str, default: Any = None) -> Any: + if isinstance(obj, Mapping): + return obj.get(name, default) + return getattr(obj, name, default) + + +def _is_int(value: Any) -> bool: + """True for a real integer; ``bool`` never counts as one here.""" + return isinstance(value, int) and not isinstance(value, bool) + + +def _shape_file_part(part: "FilePart | Mapping[str, Any]") -> dict[str, Any]: + data = _value(part, "data") + media_type = _value(part, "media_type") or "application/octet-stream" + filename = _value(part, "filename") + resolved = _materialize_attachment(data, mime_type=media_type, filename=filename) + + if resolved is not None: + return {"type": "image_url" if resolved.is_image else "file", **resolved.multimodal_part_payload} + if str(media_type).startswith("image/"): + return {"type": "image_url", "image_url": {"url": data}} + return {"type": "file", "file": {"file_data": data, "filename": filename or "file"}} + + +def _content_part(part: "Part | Mapping[str, Any]") -> Any: + kind = _value(part, "kind") + if kind in ("text", "reasoning"): + return {"type": kind, "text": _value(part, "text", "")} + if kind == "file": + return _shape_file_part(part) + if kind == "hook": + return { + "type": "hook", + "hook_id": _value(part, "hook_id"), + "hook_type": _value(part, "hook_type"), + "status": _value(part, "status"), + "resolution": _value(part, "resolution"), + } + return part + + +def _tool_call(part: "ToolCallPart | BuiltinToolCallPart | Mapping[str, Any]") -> dict[str, Any]: + return { + "id": _value(part, "tool_call_id"), + "type": "function", + "function": { + "name": _value(part, "tool_name"), + "arguments": _value(part, "tool_args", ""), + }, + } + + +def _shape_message(message: Any) -> dict[str, Any]: + role = _value(message, "role", "assistant") + text_chunks: list[str] = [] + content_source: list[Any] = [] + calls: list[dict[str, Any]] = [] + all_text = True + for part in _value(message, "parts", []) or []: + kind = _value(part, "kind") + if kind in _TOOL_CALL_KINDS: + calls.append(_tool_call(part)) + elif kind in _CONTENT_PART_KINDS: + content_source.append(part) + if kind == "text": + text_chunks.append(_value(part, "text", "")) + else: + all_text = False + + if not content_source: + content: Any = None + elif all_text: + content = "".join(text_chunks) + else: + content = [_content_part(part) for part in content_source] + + shaped: dict[str, Any] = {"role": role, "content": content} + if calls: + shaped["tool_calls"] = calls + return shaped + + +def _tool_result_content(result: Any) -> str: + if result is None: + return "" + if isinstance(result, str): + return result + return json.dumps(result, separators=(",", ":"), default=str) + + +def _tool_result_model_input(part: "ToolResultPart | BuiltinToolReturnPart | Mapping[str, Any]") -> Any: + get_model_input = getattr(part, "get_model_input", None) + if callable(get_model_input): + return get_model_input() + if isinstance(part, Mapping) and "model_input" in part: + return part["model_input"] + return _value(part, "result") + + +def _shape_input_messages(messages: Any) -> list[dict[str, Any]]: + shaped: list[dict[str, Any]] = [] + for message in messages or []: + if _value(message, "role") == "tool": + tool_results = [ + part for part in _value(message, "parts", []) or [] if _value(part, "kind") in _TOOL_RESULT_KINDS + ] + if tool_results: + shaped.extend( + { + "role": "tool", + "tool_call_id": _value(part, "tool_call_id"), + "content": _tool_result_content(_tool_result_model_input(part)), + } + for part in tool_results + ) + continue + shaped.append(_shape_message(message)) + return shaped + + +def _normalize_finish_reason(reason: Any, shaped_message: Mapping[str, Any]) -> str: + if reason in {"tool_call", "tool_calls"}: + return "tool_calls" + if isinstance(reason, str) and reason: + return reason + return "tool_calls" if shaped_message.get("tool_calls") else "stop" + + +def _shape_completion_output(message: Any, finish_reason: Any = None) -> list[dict[str, Any]] | None: + if message is None: + return None + shaped = _shape_message(message) + return [ + { + "index": 0, + "finish_reason": _normalize_finish_reason(finish_reason, shaped), + "message": shaped, + } + ] + + +def _tool_definitions(names: Any) -> list[dict[str, Any]] | None: + if not names: + return None + return [{"type": "function", "function": {"name": str(name)}} for name in names] + + +def _usage_metrics(usage: Any) -> dict[str, int | float]: + parsed = _parse_openai_usage_metrics(usage, token_name_map=_USAGE_NAME_MAP, token_prefix_map=_USAGE_PREFIX_MAP) + metrics = {name: value for name, value in parsed.items() if _is_supported_metric_value(value) and value >= 0} + if "tokens" not in metrics and "prompt_tokens" in metrics and "completion_tokens" in metrics: + metrics["tokens"] = metrics["prompt_tokens"] + metrics["completion_tokens"] + return metrics + + +def _time_to_first_token(span: Any) -> float | None: + started_at = _value(span, "started_at") + if started_at is None: + return None + for event in _value(span, "events", []) or []: + if _value(event, "name") == "first_token": + event_time = _value(event, "time_ns") + if isinstance(event_time, int): + return max(0.0, (event_time - started_at) / 1_000_000_000) + return None + + +def _sampling_metadata(params: Any) -> dict[str, Any]: + metadata: dict[str, Any] = {} + sampling = _value(params, "sampling") + samplers = sampling.values() if isinstance(sampling, Mapping) else [] + for sampler in samplers: + for name in ("temperature", "top_p", "frequency_penalty", "presence_penalty"): + value = _value(sampler, name) + if _is_supported_metric_value(value): + metadata[name] = value + + max_tokens = _value(_value(params, "output"), "max_tokens") + if _is_int(max_tokens): + metadata["max_tokens"] = max_tokens + + tool_calling = _value(params, "tool_calling") + tool_choice = _value(tool_calling, "tool_choice") + if tool_choice is not None: + choice_string = str(_value(tool_choice, "value", tool_choice)) + if choice_string in {"auto", "none", "required"}: + metadata["tool_choice"] = choice_string + elif choice_string: + metadata["tool_choice"] = {"type": "function", "function": {"name": choice_string}} + parallel_tool_calls = _value(tool_calling, "parallel_tool_calls") + if isinstance(parallel_tool_calls, bool): + metadata["parallel_tool_calls"] = parallel_tool_calls + max_tool_calls = _value(tool_calling, "max_tool_calls") + if _is_int(max_tool_calls): + metadata["max_tool_calls"] = max_tool_calls + return metadata + + +def _request_metadata(data: Any) -> dict[str, Any]: + """Metadata known when the call is issued.""" + optional = { + "tools": _tool_definitions(_value(data, "tool_names")), + "output_type": _value(data, "output_type"), + } + return { + "model": _value(data, "model") or "unknown", + "provider": _value(data, "provider") or "unknown", + **_sampling_metadata(_value(data, "params")), + **{name: value for name, value in optional.items() if value}, + } + + +def _response_metadata(data: Any) -> dict[str, Any]: + """Metadata the provider only fills in once the call completes.""" + optional = { + "model": _value(data, "response_model"), + "response_id": _value(data, "response_id"), + "finish_reason": _value(data, "finish_reason"), + } + return {name: value for name, value in optional.items() if value} + + +_FieldsBuilder = Callable[[Any, Any], dict[str, Any]] + + +def _no_fields(_upstream_span: Any, _data: Any) -> dict[str, Any]: + return {} + + +@dataclass(frozen=True) +class _SpanState: + start: _FieldsBuilder + end: _FieldsBuilder = _no_fields + + +@dataclass(frozen=True) +class _ActiveSpan: + span: Span + state: _SpanState + + +def _run_start(_upstream_span: Any, data: Any) -> dict[str, Any]: + return { + "name": "Agent.run", + "type": SpanTypeAttribute.TASK, + "input": _shape_input_messages(_value(data, "messages")), + "metadata": { + "agent": _value(data, "agent"), + "model": _value(data, "model") or "unknown", + "provider": _value(data, "provider") or "unknown", + "tool_names": _value(data, "tool_names"), + "output_type": _value(data, "output_type"), + }, + } + + +def _run_end(_upstream_span: Any, data: Any) -> dict[str, Any]: + final_message = _value(data, "final_message") + return { + "output": _shape_message(final_message) if final_message else None, + "metadata": {"blocked": bool(_value(data, "blocked"))}, + } + + +def _loop_turn_start(_upstream_span: Any, _data: Any) -> dict[str, Any]: + return {"name": "Loop Turn", "type": SpanTypeAttribute.TASK} + + +def _stream_start(_upstream_span: Any, data: Any) -> dict[str, Any]: + # The AI SDK delegates transport to an instrumentable provider client. + # Keep this framework-level span a task and let that provider own the + # single LLM span for the request. + return { + "name": "ai.stream", + "type": SpanTypeAttribute.TASK, + "input": _shape_input_messages(_value(data, "messages")), + "metadata": _request_metadata(data), + } + + +def _stream_end(upstream_span: Any, data: Any) -> dict[str, Any]: + event = { + "output": _shape_completion_output(_value(data, "message"), _value(data, "finish_reason")), + "metadata": _response_metadata(data), + } + first_token = _time_to_first_token(upstream_span) + if first_token is not None: + event["metrics"] = {"time_to_first_token": first_token} + return event + + +def _generate_start(_upstream_span: Any, data: Any) -> dict[str, Any]: + return { + "name": "ai.generate", + "type": SpanTypeAttribute.LLM, + "input": _shape_input_messages(_value(data, "messages")), + "metadata": _request_metadata(data), + } + + +def _generate_end(_upstream_span: Any, data: Any) -> dict[str, Any]: + return { + "output": _shape_completion_output(_value(data, "message")), + "metadata": _response_metadata(data), + "metrics": _usage_metrics(_value(data, "usage")), + } + + +def _tool_start(_upstream_span: Any, data: Any) -> dict[str, Any]: + return { + "name": _value(data, "tool_name") or "tool", + "type": SpanTypeAttribute.TOOL, + "input": _value(data, "args"), + "metadata": clean_nones( + { + "tool_name": _value(data, "tool_name"), + "tool_call_id": _value(data, "tool_call_id"), + "description": _value(data, "tool_description"), + } + ), + } + + +def _tool_end(upstream_span: Any, data: Any) -> dict[str, Any]: + result = _value(data, "result") + model_input = _value(data, "model_input") + event = {"output": result} + if model_input is not None: + event["metadata"] = {"model_input": model_input} + if _value(data, "is_error") and _value(upstream_span, "error") is None: + event["error"] = str(result) + return event + + +def _hook_start(_upstream_span: Any, data: Any) -> dict[str, Any]: + return { + "name": f"Hook: {_value(data, 'label')}", + "type": SpanTypeAttribute.TASK, + "input": _value(data, "metadata"), + "metadata": { + "hook_type": _value(data, "hook_type"), + "tool_call_id": _value(data, "tool_call_id"), + "status": _value(data, "status"), + }, + } + + +def _hook_end(_upstream_span: Any, data: Any) -> dict[str, Any]: + status = _value(data, "status") + return { + "output": {"status": status, "resolution": _value(data, "resolution")}, + "metadata": {"status": status}, + } + + +def _custom_start(upstream_span: Any, data: Any) -> dict[str, Any]: + return { + "name": _value(upstream_span, "name") or "custom", + "metadata": _value(data, "attrs", {}), + } + + +def _custom_end(_upstream_span: Any, data: Any) -> dict[str, Any]: + return {"metadata": _value(data, "attrs", {})} + + +def _default_start(upstream_span: Any, data: Any) -> dict[str, Any]: + kind = _value(data, "kind", _value(upstream_span, "name")) + return {"name": _value(upstream_span, "name") or str(kind or "ai")} + + +_SPAN_STATES = { + "run": _SpanState(_run_start, _run_end), + "loop_turn": _SpanState(_loop_turn_start), + "ai_stream": _SpanState(_stream_start, _stream_end), + "ai_generate": _SpanState(_generate_start, _generate_end), + "tool_execution": _SpanState(_tool_start, _tool_end), + "hook": _SpanState(_hook_start, _hook_end), + "custom": _SpanState(_custom_start, _custom_end), +} +_DEFAULT_STATE = _SpanState(_default_start) + + +def _state_for(upstream_span: Any) -> tuple[Any, _SpanState]: + data = _value(upstream_span, "data") + kind = _value(data, "kind", _value(upstream_span, "name")) + state = _SPAN_STATES.get(kind) if isinstance(kind, str) else None + return data, state or _DEFAULT_STATE + + +def _is_empty(value: Any) -> bool: + """Empty for logging purposes: absent, or an empty container. + + Deliberately keeps falsy scalars, so a tool that returns ``0`` or ``""`` + still logs an output. + """ + return value is None or (isinstance(value, (dict, list)) and not value) + + +def _end_fields(upstream_span: Any, data: Any, state: _SpanState) -> dict[str, Any]: + event = state.end(upstream_span, data) + error = _value(upstream_span, "error") + if error is not None: + # Upstream models the failure as data, not a live exception, so log the + # text rather than fabricating an exception to be stringified. + event["error"] = f"{_value(error, 'type', 'Error')}: {_value(error, 'message', str(error))}" + return {key: value for key, value in event.items() if not _is_empty(value)} + + +class BraintrustAISDKAdapter: + """Adapter for ``ai.experimental_telemetry``'s callback protocol.""" + + def __init__(self) -> None: + self._active: dict[str, _ActiveSpan] = {} + self._archived_parents: OrderedDict[str, Span] = OrderedDict() + + def _remember_parent(self, upstream_id: str, span: Span) -> None: + # ``Span.start_span`` only reads ids, which stay valid after ``end()``, + # so a finished parent still links children losslessly. + self._archived_parents[upstream_id] = span + if len(self._archived_parents) > _MAX_ARCHIVED_PARENTS: + self._archived_parents.popitem(last=False) + + async def on_span_start(self, upstream_span: Any) -> None: + upstream_id = _value(upstream_span, "id") + parent_id = _value(upstream_span, "parent_id") + start_time_ns = _value(upstream_span, "started_at") + set_as_current = bool(_value(upstream_span, "set_as_current", True)) + data, state = _state_for(upstream_span) + active_parent = self._active.get(parent_id) + parent_span = active_parent.span if active_parent is not None else self._archived_parents.get(parent_id) + kwargs = { + **state.start(upstream_span, data), + "start_time": start_time_ns / 1_000_000_000 if isinstance(start_time_ns, int) else None, + "set_current": set_as_current, + "internal": {"instrumentation": _INSTRUMENTATION}, + } + bt_span = parent_span.start_span(**kwargs) if parent_span is not None else start_span(**kwargs) + if set_as_current: + bt_span.set_current() + self._active[upstream_id] = _ActiveSpan(bt_span, state) + + async def on_span_event(self, upstream_span: Any, event: Any) -> None: + # Events remain on the upstream span snapshot. They are interpreted at + # span end so durable/replayed spans and live spans share one code path. + del upstream_span, event + + async def on_span_end(self, upstream_span: Any) -> None: + upstream_id = _value(upstream_span, "id") + active_span = self._active.pop(upstream_id, None) + if active_span is None: + return + data = _value(upstream_span, "data") + self._remember_parent(upstream_id, active_span.span) + event = _end_fields(upstream_span, data, active_span.state) + if event: + active_span.span.log(**event) + active_span.span.unset_current() + ended_at = _value(upstream_span, "ended_at") + active_span.span.end(end_time=ended_at / 1_000_000_000 if isinstance(ended_at, int) else None) diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_ai_sdk.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_ai_sdk.py new file mode 100644 index 00000000..a221e6c6 --- /dev/null +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_ai_sdk.py @@ -0,0 +1,39 @@ +"""Verify Vercel AI SDK instrumentation when ai is imported first.""" +# pylint: disable=import-error,no-member + +import asyncio + +import ai +from braintrust.auto import auto_instrument +from braintrust.integrations.test_utils import autoinstrument_test_context + + +async def main(): + results = auto_instrument() + assert results.get("ai_sdk") is True + assert auto_instrument().get("ai_sdk") is True + + with autoinstrument_test_context("test_auto_ai_sdk", integration="ai_sdk") as memory_logger: + async with ai.stream( + ai.get_model("openai:gpt-4o-mini"), + [ai.user_message("Reply with the single word hello.")], + ) as stream: + async for _ in stream: + pass + + spans = memory_logger.pop() + assert len(spans) == 2, f"Expected AI SDK and provider spans, got: {spans!r}" + ai_span = next(span for span in spans if span["span_attributes"]["name"] == "ai.stream") + provider_span = next(span for span in spans if span["span_attributes"].get("type") == "llm") + assert ai_span["span_attributes"]["type"] == "task" + for token_metric in ("tokens", "prompt_tokens", "completion_tokens"): + assert token_metric not in ai_span["metrics"] + assert provider_span["metrics"]["tokens"] > 0 + assert provider_span["metrics"]["completion_reasoning_tokens"] >= 0 + assert provider_span["metrics"]["prompt_cached_tokens"] >= 0 + assert ai_span["context"]["span_origin"]["instrumentation"]["name"] == "ai-sdk-auto" + assert provider_span["context"]["span_origin"]["instrumentation"]["name"] == "openai-auto" + + +asyncio.run(main()) +print("SUCCESS") diff --git a/py/src/braintrust/type_tests/test_public_exports.py b/py/src/braintrust/type_tests/test_public_exports.py index 19f24e89..928f6716 100644 --- a/py/src/braintrust/type_tests/test_public_exports.py +++ b/py/src/braintrust/type_tests/test_public_exports.py @@ -11,6 +11,7 @@ import pytest from braintrust import ( auto_instrument, + setup_ai_sdk, setup_pydantic_ai, wrap_anthropic, wrap_litellm, @@ -25,6 +26,7 @@ ("wrap_litellm", wrap_litellm), ("wrap_openai", wrap_openai), ("wrap_openrouter", wrap_openrouter), + ("setup_ai_sdk", setup_ai_sdk), ("setup_pydantic_ai", setup_pydantic_ai), ] diff --git a/py/uv.lock b/py/uv.lock index 6f8bf8f4..853ebfd6 100644 --- a/py/uv.lock +++ b/py/uv.lock @@ -102,6 +102,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/89/74/fdeb7c57543add6d10b68ffbfc16dc932eae72042064ed2afefdf4395b1b/agnoctl-0.1.3-py3-none-any.whl", hash = "sha256:94e1570cf2673ace2d7fa347b51c5fb2416d6f993a0a24b4fca71b04b15e72dd", size = 73695, upload-time = "2026-07-17T09:22:56.923Z" }, ] +[[package]] +name = "ai" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx", marker = "python_full_version >= '3.12'" }, + { name = "modelsdotdev", marker = "python_full_version >= '3.12'" }, + { name = "pydantic", marker = "python_full_version >= '3.12'" }, + { name = "typing-extensions", marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/e2/134605bd59c7c45a483c61572bf87dd887271ec82f3a1faf091ad91c73a0/ai-0.4.2.tar.gz", hash = "sha256:1c07429e1653d83a3b1b293bea3b2eb301c93e31439e3bca5e81fab87dc8d5a2", size = 1178023, upload-time = "2026-08-05T16:54:32.543Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/8a/7f7e48978ae40215f260e19517161b153b32ad6afc22df06a8bc8c1fcbb2/ai-0.4.2-py3-none-any.whl", hash = "sha256:55bd0d35213e202fd60632b788c29f3556e86ab8face1d9216ccdb224a1b21f0", size = 179861, upload-time = "2026-08-05T16:54:30.889Z" }, +] + [[package]] name = "aiofile" version = "3.9.0" @@ -727,6 +742,7 @@ dev = [ lint = [ { name = "agentscope" }, { name = "agno" }, + { name = "ai", marker = "python_full_version >= '3.12'" }, { name = "anthropic" }, { name = "autoevals" }, { name = "autogen-agentchat" }, @@ -943,6 +959,7 @@ dev = [ lint = [ { name = "agentscope", specifier = "<2" }, { name = "agno" }, + { name = "ai", marker = "python_full_version >= '3.12'", specifier = ">=0.4.0" }, { name = "anthropic" }, { name = "autoevals" }, { name = "autogen-agentchat" }, @@ -3937,6 +3954,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ff/6e/11d537beb67fd7dc0f1028c4a9f4f8230c6328abf6e2f2c9d0447ea0e4a4/mistralai-2.9.1-py3-none-any.whl", hash = "sha256:6f83177b8c4fdffdd2a054cf9d96fc7c0d7a474e856ecd1f0b7070344128516b", size = 1270991, upload-time = "2026-08-04T16:23:43.001Z" }, ] +[[package]] +name = "modelsdotdev" +version = "0.20260610.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/56/1505bb4d88d8c30261a2402c28ef9a7c8898bd428259ce9857931fc00603/modelsdotdev-0.20260610.0.tar.gz", hash = "sha256:f5e2d4e4c9b0a361d3e15d85e2c0fbacc787a7bdc66905807626953f3e22df9a", size = 815116, upload-time = "2026-06-10T16:38:42.015Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/0b/8cc67217511c8cbb4f997095b9abe7742627b412a1baf021a5991739a20a/modelsdotdev-0.20260610.0-py3-none-any.whl", hash = "sha256:d2595fcd3845d10b67b341d1f68b6cf25f573e89bf6f3a1e00bf81a816b909b4", size = 821929, upload-time = "2026-06-10T16:38:40.748Z" }, +] + [[package]] name = "more-itertools" version = "11.1.0"