diff --git a/examples/README.md b/examples/README.md index b7ec21e8..c9bb52e2 100644 --- a/examples/README.md +++ b/examples/README.md @@ -69,6 +69,7 @@ Unless noted otherwise, every example below uses `braintrust.auto_instrument()`. | `dspy/` | DSPy `ReAct` agent with two tools (LiteLLM token metrics propagate) | | `evals/` | The `Eval` framework — does **not** use `auto_instrument()` | | `google_genai/` | Google GenAI `generate_content` against Gemini | +| `harbor/` | Native Harbor evaluation plugin — does **not** use `auto_instrument()` | | `langchain/` | LangChain `prompt | model` chain — global handler installed by `auto_instrument()` | | `langsmith/` | Migration helper for projects coming from LangSmith — uses `setup_langsmith()` | | `litellm/` | LiteLLM `completion` | diff --git a/examples/harbor/.env.example b/examples/harbor/.env.example new file mode 100644 index 00000000..b55d5af3 --- /dev/null +++ b/examples/harbor/.env.example @@ -0,0 +1,3 @@ +BRAINTRUST_API_KEY= +OPENAI_API_KEY= +HARBOR_BRAINTRUST_PROJECT=example-harbor diff --git a/examples/harbor/.gitignore b/examples/harbor/.gitignore new file mode 100644 index 00000000..1c18760e --- /dev/null +++ b/examples/harbor/.gitignore @@ -0,0 +1 @@ +jobs/ diff --git a/examples/harbor/README.md b/examples/harbor/README.md new file mode 100644 index 00000000..a7813a98 --- /dev/null +++ b/examples/harbor/README.md @@ -0,0 +1,60 @@ +# Harbor + Braintrust + +Runs a small, self-contained [Harbor](https://harborframework.com/) evaluation and uses Harbor's native Braintrust job plugin to sync the result. Braintrust receives a managed dataset, an experiment row for the final trial, verifier rewards, and the Harbor lifecycle and ATIF trace. + +The plugin is discovered automatically through Harbor's `braintrust` entry point. The Braintrust API key remains in the host process; it is not passed into the task container. + +## Setup + +Install the example's dependencies: + +```bash +uv sync +``` + +The command below reads credentials from the repository's root `.env`. It requires: + +```dotenv +BRAINTRUST_API_KEY=... +OPENAI_API_KEY=... +``` + +Alternatively, copy `.env.example` to `.env` in this directory and change `--env-file ../../.env` below to `--env-file .env`. + +## Run + +Docker must be running. From this directory, run: + +```bash +uv run harbor run \ + --path task \ + --agent terminus-2 \ + --model openai/gpt-4.1-mini \ + --job-name braintrust-harbor-example \ + --jobs-dir jobs \ + --env-file ../../.env \ + --plugin braintrust \ + --plugin-kwarg project_name=example-harbor \ + --yes +``` + +The agent solves the task in `task/`, and Harbor's verifier emits a normalized `reward` plus an `answer_length` metric. The plugin creates `jobs/braintrust-harbor-example/braintrust-sync.json` after synchronization. + +Harbor also accepts plugin options through `HARBOR_BRAINTRUST_*` variables. For example, setting this in `.env` removes the need for the `project_name` plugin argument: + +```dotenv +HARBOR_BRAINTRUST_PROJECT=example-harbor +``` + +Then omit `--plugin-kwarg project_name=example-harbor` from the command. + +## Backfill an existing job + +To synchronize the persisted job again without rerunning the agent or verifier: + +```bash +uv run --env-file ../../.env python backfill.py jobs/braintrust-harbor-example \ + --project example-harbor +``` + +Backfill uses the same deterministic dataset, experiment, and span identities, so it reconciles the existing Braintrust data instead of creating duplicate rows. diff --git a/examples/harbor/backfill.py b/examples/harbor/backfill.py new file mode 100755 index 00000000..69baa0cc --- /dev/null +++ b/examples/harbor/backfill.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Backfill a persisted Harbor job into Braintrust.""" + +import argparse +import asyncio +from pathlib import Path + +from braintrust.integrations.harbor import backfill_job + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("job_dir", type=Path, help="Persisted Harbor job directory") + parser.add_argument("--project", help="Braintrust project name (otherwise read from the environment)") + args = parser.parse_args() + + options = {"project_name": args.project} if args.project else {} + asyncio.run(backfill_job(args.job_dir, **options)) + + +if __name__ == "__main__": + main() diff --git a/examples/harbor/pyproject.toml b/examples/harbor/pyproject.toml new file mode 100644 index 00000000..c0a26474 --- /dev/null +++ b/examples/harbor/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "braintrust-harbor-example" +version = "0.1.0" +description = "Run a Harbor evaluation and sync it to Braintrust" +requires-python = ">=3.12" +dependencies = [ + "braintrust", + "harbor==0.20.0", +] + +[tool.uv.sources] +braintrust = { path = "../../py", editable = true } diff --git a/examples/harbor/task/environment/Dockerfile b/examples/harbor/task/environment/Dockerfile new file mode 100644 index 00000000..ddf0680c --- /dev/null +++ b/examples/harbor/task/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM ubuntu:24.04 + +WORKDIR /app diff --git a/examples/harbor/task/instruction.md b/examples/harbor/task/instruction.md new file mode 100644 index 00000000..ad4d0dd7 --- /dev/null +++ b/examples/harbor/task/instruction.md @@ -0,0 +1 @@ +Calculate 17 × 6. Create `/app/answer.txt` containing only the decimal result and a trailing newline. diff --git a/examples/harbor/task/task.toml b/examples/harbor/task/task.toml new file mode 100644 index 00000000..146d31f8 --- /dev/null +++ b/examples/harbor/task/task.toml @@ -0,0 +1,24 @@ +schema_version = "1.3" +artifacts = [] + +[metadata] +category = "arithmetic" + +[verifier] +timeout_sec = 60.0 +collect = [] + +[verifier.env] + +[agent] +timeout_sec = 300.0 + +[environment] +network_mode = "public" +build_timeout_sec = 300.0 +os = "linux" +mcp_servers = [] + +[environment.env] + +[solution.env] diff --git a/examples/harbor/task/tests/test.sh b/examples/harbor/task/tests/test.sh new file mode 100755 index 00000000..1901adbf --- /dev/null +++ b/examples/harbor/task/tests/test.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +answer="$(tr -d '[:space:]' < /app/answer.txt 2>/dev/null || true)" +if [ "$answer" = "102" ]; then + reward=1 +else + reward=0 +fi + +printf '{"reward":%s,"answer_length":%s}\n' "$reward" "${#answer}" > /logs/verifier/reward.json diff --git a/py/noxfile.py b/py/noxfile.py index 60c44805..3567d533 100644 --- a/py/noxfile.py +++ b/py/noxfile.py @@ -649,6 +649,19 @@ def test_temporal(session, version): _run_tests(session, f"{INTEGRATION_DIR}/temporal") +HARBOR_VERSIONS = _get_matrix_versions("harbor") + + +@nox.session() +@nox.parametrize("version", HARBOR_VERSIONS, ids=HARBOR_VERSIONS) +def test_harbor(session, version): + if Version(platform.python_version()) < Version("3.12"): + session.skip("Harbor requires Python 3.12+") + _install_test_deps(session) + _install_matrix_dep(session, "harbor", version) + _run_tests(session, f"{INTEGRATION_DIR}/harbor", version=version) + + PYTEST_VERSIONS = _get_matrix_versions("pytest-matrix") diff --git a/py/pyproject.toml b/py/pyproject.toml index 0787f36a..9a34b7ba 100644 --- a/py/pyproject.toml +++ b/py/pyproject.toml @@ -42,6 +42,9 @@ braintrust = "braintrust.cli.__main__:main" [project.entry-points.pytest11] braintrust = "braintrust.wrappers.pytest_plugin.plugin" +[project.entry-points."harbor.plugins"] +braintrust = "braintrust.integrations.harbor:HarborPlugin" + [project.optional-dependencies] cli = ["boto3", "python-dotenv", "uv", "starlette", "uvicorn"] # TODO: remove the doc extra in the next major version. @@ -468,6 +471,14 @@ latest = "temporalio==1.31.0" "1.20.0" = "temporalio==1.20.0" "1.19.0" = "temporalio==1.19.0" +[tool.braintrust.matrix.harbor] +latest = "harbor==0.20.0" +# 0.16.0 is the oldest supported release: it is the first with +# TrialEvent.AGENT_END, which the lifecycle state machine subscribes to. Harbor +# gained its plugin system in 0.13.2, so 0.13.2-0.15.0 can load the plugin but +# cannot report the end of the agent phase. +"0.16.0" = "harbor==0.16.0" + [tool.braintrust.matrix.pytest-matrix] # Canonical pytest pin. The matching entry in [dependency-groups].test is # kept in sync by py/scripts/sync-pytest-pin.py (enforced by pre-commit). @@ -515,6 +526,7 @@ crewai = ["crewai"] dspy = ["dspy"] google_genai = ["google-genai"] huggingface_hub = ["huggingface-hub"] +harbor = ["harbor"] instructor = ["instructor"] langchain = ["langchain-core", "deepagents"] litellm = ["litellm"] @@ -538,6 +550,7 @@ cohere = "cohere" autoevals = "autoevals" braintrust-core = "braintrust_core" boto3 = "boto3" +harbor = "harbor" botocore = "botocore" crewai = "crewai" dspy = "dspy" diff --git a/py/src/braintrust/integrations/harbor/__init__.py b/py/src/braintrust/integrations/harbor/__init__.py new file mode 100644 index 00000000..54a75f71 --- /dev/null +++ b/py/src/braintrust/integrations/harbor/__init__.py @@ -0,0 +1,10 @@ +"""Braintrust's native Harbor job plugin. + +Harbor is optional. Importing this module does not import Harbor; the package is +only required when Harbor constructs the plugin or backfill reads Harbor models. +""" + +from .plugin import HarborPlugin, backfill_job + + +__all__ = ["HarborPlugin", "backfill_job"] diff --git a/py/src/braintrust/integrations/harbor/atif.py b/py/src/braintrust/integrations/harbor/atif.py new file mode 100644 index 00000000..3b203d71 --- /dev/null +++ b/py/src/braintrust/integrations/harbor/atif.py @@ -0,0 +1,534 @@ +"""Host-side ATIF to Braintrust span conversion.""" + +import json +import math +from collections.abc import Iterable +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any + +from braintrust.logger import Attachment + +from .config import PluginConfig +from .identity import NormalizedValue, child_span_id, normalize_json + + +_INSTRUMENTATION = "braintrust.plugin.harbor" +_WARNING_LIMIT = 100 + + +class _Notes: + """Collect deduplicated conversion warnings for one trajectory import. + + Normalization warnings must reach the eval root: a silently truncated or + redacted payload is indistinguishable from a faithful one. + """ + + def __init__(self) -> None: + # An insertion-ordered dict is both the dedup index and the message list. + self._seen: dict[str, None] = {} + self._suppressed = 0 + + def add(self, message: str) -> None: + if message in self._seen: + return + if len(self._seen) >= _WARNING_LIMIT: + self._suppressed += 1 + return + self._seen[message] = None + + def extend(self, messages: Iterable[str]) -> None: + for message in messages: + self.add(message) + + def record(self, normalized: NormalizedValue, context: str) -> None: + for warning in normalized.warnings: + self.add(f"{context}: {warning}") + + def finish(self) -> tuple[str, ...]: + if self._suppressed: + return (*self._seen, f"{self._suppressed} further normalization warning(s) suppressed") + return tuple(self._seen) + + +@dataclass(frozen=True) +class ATIFImportResult: + final_message: Any = None + schema_version: str | None = None + root_extra: dict[str, Any] | None = None + warnings: tuple[str, ...] = () + repairs: tuple[str, ...] = () + imported_llm_spans: int = 0 + imported_tool_spans: int = 0 + + +def _timestamp(value: Any) -> tuple[float | None, bool]: + """Parse an ATIF timestamp, reporting whether it carried no timezone.""" + if not isinstance(value, str): + return None, False + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + # A naive value is interpreted in the host timezone, matching how the + # plugin reads Harbor's own naive job timestamps. Assuming UTC here + # instead would offset every step of a naive producer on a non-UTC host + # and collapse the whole trajectory onto one clamped instant. + return parsed.timestamp(), parsed.tzinfo is None + except (ValueError, OverflowError): + return None, False + + +def _step_times(steps: list[dict[str, Any]], start: float, end: float) -> tuple[list[float], list[str]]: + if end < start: + end = start + repairs: list[str] = [] + parsed = [_timestamp(step.get("timestamp")) for step in steps] + if any(naive for _, naive in parsed): + repairs.append("interpreted timezone-naive trajectory timestamps in the host timezone") + count = max(len(steps), 1) + result: list[float] = [] + previous = start + for index, (value, _naive) in enumerate(parsed): + if value is None: + value = start + (end - start) * index / count + repairs.append(f"step {index + 1}: interpolated missing timestamp") + clamped = min(max(value, start), end) + if clamped != value: + repairs.append(f"step {index + 1}: clamped timestamp to agent phase") + if clamped < previous: + clamped = previous + repairs.append(f"step {index + 1}: repaired non-monotonic timestamp") + result.append(clamped) + previous = clamped + return result, repairs + + +def _provider(model: str | None) -> tuple[str | None, str | None]: + if not model: + return None, None + if "/" in model: + provider, model_name = model.split("/", 1) + return provider.lower(), model_name + return "unknown", model + + +def _known_single_llm_step(agent: dict[str, Any], step: dict[str, Any], metrics: dict[str, int | float]) -> bool: + # Harbor 0.20's Terminus 2 producer creates one agent step immediately + # after each LLM interaction but omits ATIF-v1.7's llm_call_count field. + # Keep this exception producer/version-specific rather than inferring from + # token usage for arbitrary ATIF producers. + return ( + agent.get("name") == "terminus-2" + and agent.get("version") == "2.0.0" + and step.get("llm_call_count") is None + and "tokens" in metrics + ) + + +def _valid_count(value: Any) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None + + +def _valid_cost(value: Any) -> float | None: + return ( + float(value) + if isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(float(value)) + and value >= 0 + else None + ) + + +def _usage_metrics(raw: Any) -> dict[str, int | float]: + if not isinstance(raw, dict): + return {} + prompt = _valid_count(raw.get("prompt_tokens")) + completion = _valid_count(raw.get("completion_tokens")) + cached = _valid_count(raw.get("cached_tokens")) + cost = _valid_cost(raw.get("cost_usd")) + metrics: dict[str, int | float] = {} + if prompt is not None: + metrics["prompt_tokens"] = prompt + if completion is not None: + metrics["completion_tokens"] = completion + if prompt is not None and completion is not None: + metrics["tokens"] = prompt + completion + if cached is not None: + metrics["prompt_cached_tokens"] = cached + if cost is not None: + metrics["estimated_cost"] = cost + extra = raw.get("extra") + if isinstance(extra, dict): + reasoning = _valid_count(extra.get("reasoning_tokens")) + first_token_ms = _valid_cost(extra.get("time_to_first_token_ms")) + cache_write = _valid_count(extra.get("cache_write_tokens")) + if reasoning is not None: + metrics["completion_reasoning_tokens"] = reasoning + if first_token_ms is not None: + metrics["time_to_first_token"] = first_token_ms / 1000 + if cache_write is not None: + metrics["prompt_cache_creation_tokens"] = cache_write + return metrics + + +def _bounded(value: Any, config: PluginConfig, notes: _Notes, context: str) -> NormalizedValue: + """Bound one trajectory payload and record what normalization removed.""" + normalized = normalize_json( + value, + max_bytes=config.max_content_bytes, + redact_patterns=config.redact_patterns, + max_depth=10, + # Trajectory content is written inside the task sandbox, so its absolute + # paths name container files the agent read and wrote. + redact_absolute_paths=False, + ) + notes.record(normalized, context) + return normalized + + +def _content( + value: Any, + trajectory_dir: Path, + config: PluginConfig, + notes: _Notes, + context: str, +) -> tuple[Any, bool]: + if isinstance(value, str) or value is None: + bounded = _bounded(value, config, notes, context) + return bounded.value, bounded.complete + if not isinstance(value, list): + return _bounded(value, config, notes, context).value, False + result: list[Any] = [] + complete = True + trajectory_root = trajectory_dir.resolve() + for index, part in enumerate(value): + part_context = f"{context}[{index}]" + if isinstance(part, dict): + if part.get("type") == "text" and isinstance(part.get("text"), str): + text = _bounded(part["text"], config, notes, part_context) + complete = complete and text.complete + result.append({"type": "text", "text": text.value}) + continue + source = part.get("source") + if part.get("type") == "image" and isinstance(source, dict) and isinstance(source.get("path"), str): + raw_path = Path(source["path"]) + if raw_path.is_absolute(): + complete = False + notes.add(f"{part_context}: image omitted because its path escapes the trajectory directory") + result.append({"type": "text", "text": "[image omitted: absolute path]"}) + continue + path = (trajectory_dir / raw_path).resolve() + try: + path.relative_to(trajectory_root) + data = path.read_bytes() + except (OSError, ValueError): + complete = False + result.append(_bounded(part, config, notes, part_context).value) + continue + if len(data) > config.max_attachment_bytes: + complete = False + notes.add(f"{part_context}: image omitted because it exceeds max_attachment_bytes") + result.append({"type": "text", "text": "[image omitted: size limit]"}) + continue + result.append( + { + "type": "image_url", + "image_url": { + "url": Attachment( + data=data, + filename=path.name, + content_type=source.get("media_type", "application/octet-stream"), + ) + }, + } + ) + continue + complete = False + result.append(_bounded(part, config, notes, part_context).value) + return result, complete + + +def _step_observations(step: dict[str, Any]) -> dict[str, Any]: + """Index one step's tool results by the call they answer. + + ATIF scopes correlation to the step: an observation result must reference a + tool_call_id declared by the same step. Indexing across the whole trajectory + instead would let a producer that reuses a tool_call_id in a later turn + overwrite an earlier turn's result. + """ + observation = step.get("observation") + if not isinstance(observation, dict) or not isinstance(observation.get("results"), list): + return {} + return { + result["source_call_id"]: result + for result in observation["results"] + if isinstance(result, dict) and isinstance(result.get("source_call_id"), str) + } + + +def _end_time(times: list[float], index: int, phase_end: float) -> float: + if index + 1 < len(times): + return max(times[index], times[index + 1]) + return max(times[index], phase_end) + + +def summarize_trajectory(trajectory_path: Path, config: PluginConfig) -> ATIFImportResult: + """Read bounded trajectory summary data without creating detailed leaves.""" + try: + if trajectory_path.stat().st_size > config.max_total_attachment_bytes: + return ATIFImportResult(warnings=("trajectory omitted: size limit",)) + trajectory = json.loads(trajectory_path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + return ATIFImportResult(warnings=(f"trajectory unavailable or malformed: {exc}",)) + if not isinstance(trajectory, dict) or not isinstance(trajectory.get("steps"), list): + return ATIFImportResult(warnings=("trajectory malformed: steps must be an array",)) + notes = _Notes() + final_message = None + # Only the last agent step is kept, so normalize that one rather than every + # step: the discarded walks would also report warnings for messages that are + # never logged. + steps = trajectory["steps"] + for index in range(len(steps) - 1, -1, -1): + step = steps[index] + if isinstance(step, dict) and step.get("source") == "agent" and not step.get("is_copied_context"): + final_message = _bounded(step.get("message"), config, notes, f"step {index + 1} message").value + break + extra = trajectory.get("extra") if isinstance(trajectory.get("extra"), dict) else None + final_metrics = trajectory.get("final_metrics") + root_extra = dict(extra or {}) + if isinstance(final_metrics, dict): + root_extra["final_metrics"] = _bounded(final_metrics, config, notes, "final_metrics").value + return ATIFImportResult( + final_message=final_message, + schema_version=( + trajectory.get("schema_version") if isinstance(trajectory.get("schema_version"), str) else None + ), + root_extra=root_extra or None, + warnings=notes.finish(), + ) + + +def import_trajectory( + parent: Any, + trajectory_path: Path, + *, + trial_id: str, + semantic_prefix: str, + phase_start: float, + phase_end: float, + config: PluginConfig, + _trajectory_data: dict[str, Any] | None = None, +) -> ATIFImportResult: + notes = _Notes() + if _trajectory_data is not None: + trajectory = _trajectory_data + else: + try: + if trajectory_path.stat().st_size > config.max_total_attachment_bytes: + return ATIFImportResult(warnings=("trajectory omitted: size limit",)) + trajectory = json.loads(trajectory_path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + return ATIFImportResult(warnings=(f"trajectory unavailable or malformed: {exc}",)) + if not isinstance(trajectory, dict) or not isinstance(trajectory.get("steps"), list): + return ATIFImportResult(warnings=("trajectory malformed: steps must be an array",)) + + steps = [step for step in trajectory["steps"] if isinstance(step, dict)] + times, repairs = _step_times(steps, phase_start, phase_end) + agent = trajectory.get("agent") if isinstance(trajectory.get("agent"), dict) else {} + default_model = agent.get("model_name") + tools = agent.get("tool_definitions") if isinstance(agent.get("tool_definitions"), list) else None + # The tool configuration is one value for the whole trajectory. Normalize it + # once: doing it per step both repeats the work and, because each context + # names a different step, defeats warning dedup. Bounding auxiliary metadata + # is always allowed, but a truncated tool list must be omitted rather than + # logged as if it were the model's real tool configuration. + llm_tools: Any = None + if tools: + bounded_tools = _bounded(tools, config, notes, "tool definitions") + if bounded_tools.complete: + llm_tools = bounded_tools.value + else: + notes.add("tool definitions omitted after normalization") + messages: list[dict[str, Any]] = [] + final_message: Any = None + llm_count = 0 + tool_count = 0 + for index, step in enumerate(steps): + source = step.get("source") + content, content_complete = _content( + step.get("message"), trajectory_path.parent, config, notes, f"step {index + 1} message" + ) + if source in {"system", "user"}: + if config.content_mode != "metadata": + messages.append({"role": source, "content": content}) + continue + if source != "agent": + notes.add(f"step {index + 1}: unknown source") + continue + + tool_calls = step.get("tool_calls") if isinstance(step.get("tool_calls"), list) else [] + assistant_message: dict[str, Any] = {"role": "assistant", "content": content} + normalized_calls: list[dict[str, Any]] = [] + for call in tool_calls: + if not isinstance(call, dict): + continue + call_id, name, arguments = call.get("tool_call_id"), call.get("function_name"), call.get("arguments") + if isinstance(call_id, str) and isinstance(name, str) and isinstance(arguments, dict): + normalized_calls.append( + { + "id": call_id, + "type": "function", + "function": {"name": name, "arguments": json.dumps(arguments, sort_keys=True)}, + } + ) + if normalized_calls: + assistant_message["tool_calls"] = normalized_calls + + llm_call_count = step.get("llm_call_count") + metrics = _usage_metrics(step.get("metrics")) + if _known_single_llm_step(agent, step, metrics): + llm_call_count = 1 + repairs.append(f"step {index + 1}: inferred one model call from terminus-2 2.0.0 trajectory") + provider, model = _provider(step.get("model_name") or default_model) + can_be_llm = ( + config.content_mode != "metadata" + and llm_call_count == 1 + and content_complete + and provider is not None + and model is not None + and "tokens" in metrics + ) + path = f"{semantic_prefix}/turn/{step.get('step_id', index + 1)}" + if can_be_llm: + metadata: dict[str, Any] = {"provider": provider, "model": model} + if llm_tools is not None: + metadata["tools"] = llm_tools + llm_span = parent.start_span( + name="chat.completions.create", + type="llm", + id=child_span_id(trial_id, f"{path}/llm"), + start_time=times[index], + set_current=False, + input=list(messages), + metadata=metadata, + internal={"instrumentation": _INSTRUMENTATION}, + ) + llm_span.log(output=assistant_message, metrics=metrics) + llm_span.end(end_time=_end_time(times, index, phase_end)) + llm_count += 1 + else: + reason = "not exactly one conforming model call" + if llm_call_count == 1 and "tokens" not in metrics: + reason = "model call missing token usage" + elif llm_call_count == 1 and not content_complete: + reason = "message content was truncated or redacted" + notes.add(f"step {index + 1}: downgraded to task ({reason})") + summary_span = parent.start_span( + name=f"trajectory.step.{step.get('step_id', index + 1)}", + type="task", + id=child_span_id(trial_id, f"{path}/summary"), + start_time=times[index], + set_current=False, + input={"source": source}, + internal={"instrumentation": _INSTRUMENTATION}, + ) + summary_span.log(output={"message": content, "tool_call_count": len(normalized_calls)}) + summary_span.end(end_time=_end_time(times, index, phase_end)) + + if config.content_mode != "metadata": + messages.append(assistant_message) + if not step.get("is_copied_context"): + final_message = assistant_message + + observations = _step_observations(step) + for call_index, call in enumerate(tool_calls): + if not isinstance(call, dict): + continue + call_id, name, arguments = call.get("tool_call_id"), call.get("function_name"), call.get("arguments") + result = observations.get(call_id) if isinstance(call_id, str) else None + # Scope the span id to the turn, for the reason _step_observations gives. + tool_path = f"{path}/tool/{call_id or call_index}" + if ( + isinstance(call_id, str) + and isinstance(name, str) + and isinstance(arguments, dict) + and isinstance(result, dict) + ): + tool_context = f"step {index + 1} tool {call_id}" + tool_output, tool_complete = _content( + result.get("content"), trajectory_path.parent, config, notes, f"{tool_context} result" + ) + tool_input = _bounded(arguments, config, notes, f"{tool_context} arguments") + result_extra = result.get("extra") if isinstance(result.get("extra"), dict) else {} + tool_error = result_extra.get("error") if isinstance(result_extra.get("error"), str) else None + has_result = result.get("content") is not None or tool_error is not None + if config.content_mode != "metadata" and tool_complete and tool_input.complete and has_result: + tool_span = parent.start_span( + name=name, + type="tool", + id=child_span_id(trial_id, tool_path), + start_time=times[index], + set_current=False, + input=tool_input.value, + metadata={"tool_call_id": call_id}, + internal={"instrumentation": _INSTRUMENTATION}, + ) + if tool_error is not None: + tool_span.log(error=tool_error) + else: + tool_span.log(output=tool_output) + tool_span.end(end_time=_end_time(times, index, phase_end)) + tool_count += 1 + else: + notes.add(f"step {index + 1} tool {call_id}: downgraded because payload is incomplete") + if config.content_mode != "metadata": + messages.append({"role": "tool", "tool_call_id": call_id, "content": tool_output}) + else: + notes.add(f"step {index + 1} tool {call_id or call_index}: missing correlated arguments or result") + + # Preserve subagents as explicit nested task trees. Their detailed leaves use + # the same conformance gate recursively. + for sub_index, subagent in enumerate(trajectory.get("subagent_trajectories") or []): + if not isinstance(subagent, dict): + continue + sub_parent = parent.start_span( + name=f"subagent:{(subagent.get('agent') or {}).get('name', sub_index)}", + type="task", + id=child_span_id(trial_id, f"{semantic_prefix}/subagent/{sub_index}"), + start_time=phase_start, + set_current=False, + internal={"instrumentation": _INSTRUMENTATION}, + ) + imported = import_trajectory( + sub_parent, + trajectory_path, + trial_id=trial_id, + semantic_prefix=f"{semantic_prefix}/subagent/{sub_index}", + phase_start=phase_start, + phase_end=phase_end, + config=config, + _trajectory_data=subagent, + ) + sub_parent.end(end_time=phase_end) + # Step numbers restart inside a subagent, so namespace its warnings the way + # span identity is namespaced. Otherwise dedup silently drops a subagent + # warning that reads identically to one from the parent's own steps. + notes.extend(f"subagent {sub_index}: {warning}" for warning in imported.warnings) + repairs.extend(f"subagent {sub_index}: {repair}" for repair in imported.repairs) + llm_count += imported.imported_llm_spans + tool_count += imported.imported_tool_spans + + extra = trajectory.get("extra") if isinstance(trajectory.get("extra"), dict) else None + root_extra = dict(extra or {}) + if isinstance(trajectory.get("final_metrics"), dict): + root_extra["final_metrics"] = _bounded(trajectory["final_metrics"], config, notes, "final_metrics").value + return ATIFImportResult( + final_message=final_message, + schema_version=trajectory.get("schema_version") if isinstance(trajectory.get("schema_version"), str) else None, + root_extra=root_extra or None, + warnings=notes.finish(), + repairs=tuple(repairs), + imported_llm_spans=llm_count, + imported_tool_spans=tool_count, + ) diff --git a/py/src/braintrust/integrations/harbor/cassettes/0.16.0/test_atif_import_round_trips_with_real_sdks.yaml b/py/src/braintrust/integrations/harbor/cassettes/0.16.0/test_atif_import_round_trips_with_real_sdks.yaml new file mode 100644 index 00000000..fc07089b --- /dev/null +++ b/py/src/braintrust/integrations/harbor/cassettes/0.16.0/test_atif_import_round_trips_with_real_sdks.yaml @@ -0,0 +1,641 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/apikey/login + response: + body: + string: '{"org_info":[{"id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"Braintrust + SDKs","api_url":"https://api.braintrust.dev","git_metadata":{"collect":"some","fields":["commit","branch","tag","dirty","author_name","author_email","commit_message","commit_time"]},"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '376' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-OGYwNjlmNTYtMmFiOC00NjkwLWJjNGQtZDQ0ZDYwZTE5ODQ4'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 07 Aug 2026 13:58:41 GMT + Etag: + - '"13vsc5ye8flag"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/apikey/login + X-Nonce: + - OGYwNjlmNTYtMmFiOC00NjkwLWJjNGQtZDQ0ZDYwZTE5ODQ4 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::sv6gv-1786111121104-17adf5c6b5e3 + status: + code: 200 + message: OK +- request: + body: '{"project_name": "python-sdk-harbor-tests", "project_id": null, "org_id": + "5abfae3a-7aa7-4653-a9c8-b3efcb18f584", "update": true, "experiment_name": "harbor-atif-import", + "public": false}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '187' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/experiment/register + response: + body: + string: '{"project":{"id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-harbor-tests","description":null,"created":"2026-07-30T23:19:36.349Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null},"experiment":{"id":"7aba8758-862e-41ef-9e77-6247c4b71287","project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","name":"harbor-atif-import","description":null,"created":"2026-07-30T23:19:36.349Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","metadata":null,"tags":null}}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '745' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-YWJmNTI2ZjAtNzIzMC00ODljLTkyNWItMDRkNGM2MWU1Mjg1'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 07 Aug 2026 13:58:41 GMT + Etag: + - '"29xhqzk7lxkp"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/experiment/register + X-Nonce: + - YWJmNTI2ZjAtNzIzMC00ODljLTkyNWItMDRkNGM2MWU1Mjg1 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::ld757-1786111121467-97fbb22f061a + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/version + response: + body: + string: '{"version":"2.9.0","date_version":"20260805","ff_version":44,"commit":"787a5ad8b0dd2090d79c37513ba5cc3d21758ddc","deployment_mode":"lambda","deployment_type":"custom","brainstore_default":"force","brainstore_can_contain_row_refs":true,"skip_pg_config":"all","has_realtime_wal_bucket":true,"brainstore_wal_footer_version":"v3","brainstore_wal_use_efficient_format":true,"has_logs2":true,"brainstore_export_enabled":true,"js":true,"universal":true,"code_execution":true,"logs3_payload_max_bytes":5242880,"control_plane_telemetry":["status","memprof","usage"]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 07 Aug 2026 13:58:41 GMT + Via: + - 1.1 5cac25c3f8a8e7707f05f3c2c47aa970.cloudfront.net (CloudFront), 1.1 17bd0a3b88141b04bc745d7ececd22ee.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - 1FtHbLwzegI9jWKl8nNa5UuVd_JoZN1ofTVduDkrREY5X0al0Up-Tg== + X-Amz-Cf-Pop: + - IAD61-P9 + - IAD89-P4 + X-Amzn-Trace-Id: + - Root=1-6a75e491-6bf882010e0d8b42687d4e4d;Parent=5f3e19a2db548ab9;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '557' + etag: + - W/"22d-RtkjTszsylxX5VGRelXw9TnJwV8" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin + x-amz-apigw-id: + - BvCm3FrToAMENvg= + x-amzn-Remapped-content-length: + - '557' + x-amzn-RequestId: + - ee9067e0-e5ae-40b6-a8d6-6c1f335095c5 + x-bt-internal-trace-id: + - 6a75e4910000000056d068c70253f76c + status: + code: 200 + message: OK +- request: + body: '{"rows": [{"_is_merge": false, "context": {"caller_filename": "[REDACTED_PATH]", + "caller_functionname": "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": + {"instrumentation": {"name": "braintrust-python-logger"}, "name": "braintrust.sdk.python", + "version": "0.31.1"}}, "created": "2026-08-07T13:58:40.927111+00:00", "experiment_id": + "7aba8758-862e-41ef-9e77-6247c4b71287", "id": "c7a87986-0192-5f40-9ac0-a535810f1fe7", + "metrics": {"start": 1767225600.0}, "root_span_id": "aa387bfd78c8b2de2cb8c20721e65d08", + "span_attributes": {"exec_counter": 1, "name": "agent_execution", "type": "task"}, + "span_id": "00a19d81c889cf38", "span_parents": null}], "api_version": 2}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '831' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/logs3 + response: + body: + string: '{"ids":["c7a87986-0192-5f40-9ac0-a535810f1fe7"],"xact_id":"1000197646831503725"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 07 Aug 2026 13:58:42 GMT + Via: + - 1.1 874d2226efce88074e49219a95089ade.cloudfront.net (CloudFront), 1.1 2f76b89b5b812e346fc5b368361bed3c.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - NVZ1Gve_PJlggDaPmsoshIAxPhlfVo53ese99v4OxC8ynfAPX6eRGg== + X-Amz-Cf-Pop: + - IAD61-P9 + - IAD89-P4 + X-Amzn-Trace-Id: + - Root=1-6a75e492-2ce6f4756052ca0463b7f324;Parent=0241281b1d663626;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '80' + etag: + - W/"50-YN8eD/A30NESutY9PskfLVJGzQA" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - BvCm5HDSoAMEEhg= + x-amzn-RequestId: + - bcd579d1-8de9-480a-bb09-e015bab059ed + x-bt-internal-trace-id: + - 6a75e492000000001ad7dbe325c3697e + status: + code: 200 + message: OK +- request: + body: '{"rows": [{"context": {"caller_filename": "[REDACTED_PATH]", "caller_functionname": + "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": {"instrumentation": + {"name": "braintrust.plugin.harbor"}, "name": "braintrust.sdk.python", "version": + "0.31.1"}}, "created": "2026-08-07T13:58:40.927670+00:00", "experiment_id": + "7aba8758-862e-41ef-9e77-6247c4b71287", "id": "7eb93a96-dfea-5318-bfff-63cf37ce89c6", + "input": [{"content": "What is 2+2?", "role": "user"}], "metadata": {"model": + "gpt-4o-mini", "provider": "openai", "tools": [{"function": {"name": "calculator", + "parameters": {"type": "object"}}, "type": "function"}]}, "metrics": {"completion_tokens": + 4, "end": 1767225602.0, "estimated_cost": 0.001, "prompt_tokens": 10, "start": + 1767225601.0, "tokens": 14}, "output": {"content": "I''ll calculate it.", "role": + "assistant", "tool_calls": [{"function": {"arguments": "{\"expression\": \"2+2\"}", + "name": "calculator"}, "id": "call_1", "type": "function"}]}, "root_span_id": + "aa387bfd78c8b2de2cb8c20721e65d08", "span_attributes": {"exec_counter": 2, "name": + "chat.completions.create", "type": "llm"}, "span_id": "7865efa0ef2f3450", "span_parents": + ["00a19d81c889cf38"]},{"context": {"caller_filename": "[REDACTED_PATH]", "caller_functionname": + "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": {"instrumentation": + {"name": "braintrust.plugin.harbor"}, "name": "braintrust.sdk.python", "version": + "0.31.1"}}, "created": "2026-08-07T13:58:40.927957+00:00", "experiment_id": + "7aba8758-862e-41ef-9e77-6247c4b71287", "id": "b134c10f-7dc4-543a-9707-fa9af07dbbe8", + "input": {"expression": "2+2"}, "metadata": {"tool_call_id": "call_1"}, "metrics": + {"end": 1767225602.0, "start": 1767225601.0}, "output": "4", "root_span_id": + "aa387bfd78c8b2de2cb8c20721e65d08", "span_attributes": {"exec_counter": 3, "name": + "calculator", "type": "tool"}, "span_id": "a931b71567cac876", "span_parents": + ["00a19d81c889cf38"]},{"context": {"caller_filename": "[REDACTED_PATH]", "caller_functionname": + "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": {"instrumentation": + {"name": "braintrust.plugin.harbor"}, "name": "braintrust.sdk.python", "version": + "0.31.1"}}, "created": "2026-08-07T13:58:40.928537+00:00", "experiment_id": + "7aba8758-862e-41ef-9e77-6247c4b71287", "id": "a2713e08-b2cb-5762-99d7-be29120f9880", + "input": [{"content": "What is 2+2?", "role": "user"}, {"content": "I''ll calculate + it.", "role": "assistant", "tool_calls": [{"function": {"arguments": "{\"expression\": + \"2+2\"}", "name": "calculator"}, "id": "call_1", "type": "function"}]}, {"content": + "4", "role": "tool", "tool_call_id": "call_1"}], "metadata": {"model": "gpt-4o-mini", + "provider": "openai", "tools": [{"function": {"name": "calculator", "parameters": + {"type": "object"}}, "type": "function"}]}, "metrics": {"completion_tokens": + 5, "end": 1767225603.0, "prompt_tokens": 15, "start": 1767225602.0, "tokens": + 20}, "output": {"content": "The answer is 4.", "role": "assistant"}, "root_span_id": + "aa387bfd78c8b2de2cb8c20721e65d08", "span_attributes": {"exec_counter": 4, "name": + "chat.completions.create", "type": "llm"}, "span_id": "c9055218be89abad", "span_parents": + ["00a19d81c889cf38"]},{"_is_merge": true, "experiment_id": "7aba8758-862e-41ef-9e77-6247c4b71287", + "id": "c7a87986-0192-5f40-9ac0-a535810f1fe7", "metrics": {"end": 1767225603.0}, + "root_span_id": "aa387bfd78c8b2de2cb8c20721e65d08", "span_id": "00a19d81c889cf38", + "span_parents": null}], "api_version": 2}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '3935' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/logs3 + response: + body: + string: '{"ids":["7eb93a96-dfea-5318-bfff-63cf37ce89c6","b134c10f-7dc4-543a-9707-fa9af07dbbe8","a2713e08-b2cb-5762-99d7-be29120f9880","c7a87986-0192-5f40-9ac0-a535810f1fe7"],"xact_id":"1000197646831505354"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 07 Aug 2026 13:58:42 GMT + Via: + - 1.1 ba8b183ec1776a2582f8b77753f81c2a.cloudfront.net (CloudFront), 1.1 5fef2688877996791689cf17ab2832d0.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - Rwb9acU7bKFUNPl3PiBWgqsKuD0f569J9tosU-QrxYpDfsBBGmAc6Q== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a75e492-2f6a5b3b33d47b5666157d4f;Parent=787660caf4d840aa;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '197' + etag: + - W/"c5-HUmmOgIvpbdVKITvoQCHy4vcUp0" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - BvCm9FsPoAMEbjg= + x-amzn-RequestId: + - 73a8ed62-f5bd-4091-9855-3eb7c2060e1f + x-bt-internal-trace-id: + - 6a75e4920000000045361407a343fa4e + status: + code: 200 + message: OK +- request: + body: '{"query": {"select": [{"op": "star"}], "from": {"op": "function", "name": + {"op": "ident", "name": ["experiment"]}, "args": [{"op": "literal", "value": + "7aba8758-862e-41ef-9e77-6247c4b71287"}]}, "cursor": null, "limit": 1000}, "use_columnstore": + false, "brainstore_realtime": true, "query_source": "py_sdk_object_fetcher_experiment"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip + Connection: + - keep-alive + Content-Length: + - '332' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/btql + response: + body: + string: '{"data":[{"_pagination_key":"p07671288859497988098","_xact_id":"1000197646831505354","audit_data":[{"_xact_id":"1000197646831505354","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.1"}},"created":"2026-08-07T13:58:40.928Z","error":null,"expected":null,"experiment_id":"7aba8758-862e-41ef-9e77-6247c4b71287","facets":null,"id":"a2713e08-b2cb-5762-99d7-be29120f9880","input":[{"content":"What + is 2+2?","role":"user"},{"content":"I''ll calculate it.","role":"assistant","tool_calls":[{"function":{"arguments":"{\"expression\": + \"2+2\"}","name":"calculator"},"id":"call_1","type":"function"}]},{"content":"4","role":"tool","tool_call_id":"call_1"}],"is_root":false,"metadata":{"model":"gpt-4o-mini","provider":"openai","tools":[{"function":{"name":"calculator","parameters":{"type":"object"}},"type":"function"}]},"metrics":{"completion_tokens":5,"end":1767225603,"prompt_tokens":15,"start":1767225602,"tokens":20},"origin":null,"output":{"content":"The + answer is 4.","role":"assistant"},"project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"aa387bfd78c8b2de2cb8c20721e65d08","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":4,"name":"chat.completions.create","type":"llm"},"span_id":"c9055218be89abad","span_parents":["00a19d81c889cf38"],"tags":null},{"_pagination_key":"p07671288859497988097","_xact_id":"1000197646831505354","audit_data":[{"_xact_id":"1000197646831505354","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.1"}},"created":"2026-08-07T13:58:40.927Z","error":null,"expected":null,"experiment_id":"7aba8758-862e-41ef-9e77-6247c4b71287","facets":null,"id":"b134c10f-7dc4-543a-9707-fa9af07dbbe8","input":{"expression":"2+2"},"is_root":false,"metadata":{"tool_call_id":"call_1"},"metrics":{"end":1767225602,"start":1767225601},"origin":null,"output":"4","project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"aa387bfd78c8b2de2cb8c20721e65d08","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":3,"name":"calculator","type":"tool"},"span_id":"a931b71567cac876","span_parents":["00a19d81c889cf38"],"tags":null},{"_pagination_key":"p07671288859497988096","_xact_id":"1000197646831505354","audit_data":[{"_xact_id":"1000197646831505354","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.1"}},"created":"2026-08-07T13:58:40.927Z","error":null,"expected":null,"experiment_id":"7aba8758-862e-41ef-9e77-6247c4b71287","facets":null,"id":"7eb93a96-dfea-5318-bfff-63cf37ce89c6","input":[{"content":"What + is 2+2?","role":"user"}],"is_root":false,"metadata":{"model":"gpt-4o-mini","provider":"openai","tools":[{"function":{"name":"calculator","parameters":{"type":"object"}},"type":"function"}]},"metrics":{"completion_tokens":4,"end":1767225602,"estimated_cost":0.001,"prompt_tokens":10,"start":1767225601,"tokens":14},"origin":null,"output":{"content":"I''ll + calculate it.","role":"assistant","tool_calls":[{"function":{"arguments":"{\"expression\": + \"2+2\"}","name":"calculator"},"id":"call_1","type":"function"}]},"project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"aa387bfd78c8b2de2cb8c20721e65d08","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":2,"name":"chat.completions.create","type":"llm"},"span_id":"7865efa0ef2f3450","span_parents":["00a19d81c889cf38"],"tags":null},{"_pagination_key":"p07671288859391229952","_xact_id":"1000197646831505354","audit_data":[{"_xact_id":"1000197646831503725","audit_data":{"action":"upsert"},"metadata":{},"source":"api"},{"_xact_id":"1000197646831505354","audit_data":{"action":"merge","from":null,"path":["metrics"],"to":{"end":1767225603}},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust-python-logger"},"name":"braintrust.sdk.python","version":"0.31.1"}},"created":"2026-08-07T13:58:40.927Z","error":null,"expected":null,"experiment_id":"7aba8758-862e-41ef-9e77-6247c4b71287","facets":null,"id":"c7a87986-0192-5f40-9ac0-a535810f1fe7","input":null,"is_root":true,"metadata":null,"metrics":{"end":1767225603,"start":1767225600},"origin":null,"output":null,"project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"aa387bfd78c8b2de2cb8c20721e65d08","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":1,"name":"agent_execution","type":"task"},"span_id":"00a19d81c889cf38","span_parents":null,"tags":null},{"_pagination_key":"p07668465162241900545","_xact_id":"1000197603745304547","audit_data":[{"_xact_id":"1000197603738631632","audit_data":{"action":"upsert"},"metadata":{},"source":"api"},{"_xact_id":"1000197603742781733","audit_data":{"action":"upsert"},"metadata":{},"source":"api"},{"_xact_id":"1000197603745304547","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.0"}},"created":"2026-07-30T23:21:18.801Z","error":null,"expected":null,"experiment_id":"7aba8758-862e-41ef-9e77-6247c4b71287","facets":null,"id":"843c7c39-2f3a-5813-aa42-56da9fe5709f","input":{"expression":"2+2"},"is_root":false,"metadata":{"tool_call_id":"call_1"},"metrics":{"end":1767225602,"start":1767225601},"origin":null,"output":"4","project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"d43785839001054eaf1c4ac196c82112","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":3,"name":"calculator","type":"tool"},"span_id":"161550719b4ad9d1","span_parents":["2d261ed7ec06b709"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"_pagination_key":{"description":"A + stable, time-ordered key that can be used to paginate over experiment events. + This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The + transaction id of an event is unique to the network operation that processed + the event insertion. Transaction ids are monotonically increasing over time + and can be used to retrieve a versioned snapshot of the experiment (see the + `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"additionalProperties":false,"properties":{"confidence":{"description":"Optional + confidence score for the classification","type":["number","null"]},"id":{"description":"Stable + classification identifier","type":"string"},"label":{"description":"Original + label of the classification item, which is useful for search and indexing + purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional + metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The + version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The + type of global function. Defaults to ''scorer''.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional + function identifier that produced the classification"}},"required":["id"],"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name + of the file in code where the experiment event was created","type":["string","null"]},"caller_functionname":{"description":"The + function in code which created the experiment event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The + timestamp the experiment event was created","format":"date-time","type":"string"},"error":{"description":"The + error that occurred, if any."},"expected":{"description":"The ground truth + value (an arbitrary, JSON serializable object) that you''d compare to `output` + to determine if your `output` value is correct or not. Braintrust currently + does not compare `output` to `expected` for you, since there are so many different + ways to do that correctly. Instead, these values are just used to help you + navigate your experiments while digging into analyses. However, we may later + use these values to re-score outputs or fine-tune your models"},"experiment_id":{"description":"Unique + identifier for the experiment","format":"uuid","type":"string"},"facets":{"anyOf":[{"additionalProperties":{"type":["string","null"]},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A + unique identifier for the experiment event. If you don''t provide one, Braintrust + will generate one for you","type":"string"},"input":{"description":"The arguments + that uniquely define a test case (an arbitrary, JSON serializable object). + Later on, Braintrust will use the `input` to know whether two test cases are + the same between experiments, so they should not contain experiment-specific + state. A simple rule of thumb is that if you run the same experiment twice, + the `input` should be identical"},"is_root":{"description":"Whether this span + is a root span","type":["boolean","null"]},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The + model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This + metric is deprecated"},"caller_functionname":{"description":"This metric is + deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A + unix timestamp recording when the section of code which produced the experiment + event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A + unix timestamp recording when the section of code which produced the experiment + event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"origin":{"anyOf":[{"description":"Reference + to the original object and event this was copied from.","properties":{"_xact_id":{"description":"Transaction + ID of the original event.","type":["string","null"]},"created":{"description":"Created + timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID + of the original event.","type":"string"},"object_id":{"description":"ID of + the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type + of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"required":["object_type","object_id","id"],"type":"object"},{"type":"null"}]},"output":{"description":"The + output of your application, including post-processing (an arbitrary, JSON + serializable object), that allows you to determine whether the result is correct + or not. For example, in an app that generates SQL queries, the `output` should + be the _result_ of the SQL query generated by the model, not the query itself, + because there may be multiple valid queries that answer a single question"},"project_id":{"description":"Unique + identifier for the project that the experiment belongs under","format":"uuid","type":"string"},"root_span_id":{"description":"A + unique identifier for the trace this experiment event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"additionalProperties":{},"description":"Human-identifying + attributes of the span, such as name, type, etc.","properties":{"name":{"description":"Name + of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A + unique identifier used to link different experiment events together as part + of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) + for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}}},"cursor":"amvcb4/jAAE","realtime_state":{"type":"on","minimum_xact_id":"1000197642012143506","read_bytes":5773,"actual_xact_id":"1000197646831505354"},"freshness_state":{"last_processed_xact_id":"1000197642012143506","last_considered_xact_id":"1000197646831505354"},"warnings":[]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Fri, 07 Aug 2026 13:58:43 GMT + Via: + - 1.1 54704f25530eb7440881e3a166d68472.cloudfront.net (CloudFront), 1.1 d03af248468c898a111754f0666c2316.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - GEuFZsCyR21ehisxpE3RuZBhz3jECh8YoWPgy088TXrDEO08TKzOkQ== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a75e492-72746caa06a291d4459be01c;Parent=53ed2c89a9d62dda;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - private, no-cache + content-length: + - '15687' + vary: + - Origin + x-amz-apigw-id: + - BvCnAEu-oAMEMVw= + x-amzn-RequestId: + - b301bde2-2699-4076-8193-01ea71b2d1d9 + x-bt-api-duration-ms: + - '452' + x-bt-brainstore-duration-ms: + - '406' + x-bt-cursor: + - amvcb4/jAAE + x-bt-internal-trace-id: + - 6a75e492000000001bee271c77fdaac5 + status: + code: 200 + message: OK +- request: + body: '{"query": {"select": [{"op": "star"}], "from": {"op": "function", "name": + {"op": "ident", "name": ["experiment"]}, "args": [{"op": "literal", "value": + "7aba8758-862e-41ef-9e77-6247c4b71287"}]}, "cursor": "amvcb4/jAAE", "limit": + 1000}, "use_columnstore": false, "brainstore_realtime": true, "query_source": + "py_sdk_object_fetcher_experiment"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip + Connection: + - keep-alive + Content-Length: + - '341' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/btql + response: + body: + string: '{"data":[],"schema":{"type":"array","items":{"type":"object","properties":{"_pagination_key":{"description":"A + stable, time-ordered key that can be used to paginate over experiment events. + This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The + transaction id of an event is unique to the network operation that processed + the event insertion. Transaction ids are monotonically increasing over time + and can be used to retrieve a versioned snapshot of the experiment (see the + `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"additionalProperties":false,"properties":{"confidence":{"description":"Optional + confidence score for the classification","type":["number","null"]},"id":{"description":"Stable + classification identifier","type":"string"},"label":{"description":"Original + label of the classification item, which is useful for search and indexing + purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional + metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The + version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The + type of global function. Defaults to ''scorer''.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional + function identifier that produced the classification"}},"required":["id"],"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name + of the file in code where the experiment event was created","type":["string","null"]},"caller_functionname":{"description":"The + function in code which created the experiment event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The + timestamp the experiment event was created","format":"date-time","type":"string"},"error":{"description":"The + error that occurred, if any."},"expected":{"description":"The ground truth + value (an arbitrary, JSON serializable object) that you''d compare to `output` + to determine if your `output` value is correct or not. Braintrust currently + does not compare `output` to `expected` for you, since there are so many different + ways to do that correctly. Instead, these values are just used to help you + navigate your experiments while digging into analyses. However, we may later + use these values to re-score outputs or fine-tune your models"},"experiment_id":{"description":"Unique + identifier for the experiment","format":"uuid","type":"string"},"facets":{"anyOf":[{"additionalProperties":{"type":["string","null"]},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A + unique identifier for the experiment event. If you don''t provide one, Braintrust + will generate one for you","type":"string"},"input":{"description":"The arguments + that uniquely define a test case (an arbitrary, JSON serializable object). + Later on, Braintrust will use the `input` to know whether two test cases are + the same between experiments, so they should not contain experiment-specific + state. A simple rule of thumb is that if you run the same experiment twice, + the `input` should be identical"},"is_root":{"description":"Whether this span + is a root span","type":["boolean","null"]},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The + model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This + metric is deprecated"},"caller_functionname":{"description":"This metric is + deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A + unix timestamp recording when the section of code which produced the experiment + event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A + unix timestamp recording when the section of code which produced the experiment + event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"origin":{"anyOf":[{"description":"Reference + to the original object and event this was copied from.","properties":{"_xact_id":{"description":"Transaction + ID of the original event.","type":["string","null"]},"created":{"description":"Created + timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID + of the original event.","type":"string"},"object_id":{"description":"ID of + the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type + of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"required":["object_type","object_id","id"],"type":"object"},{"type":"null"}]},"output":{"description":"The + output of your application, including post-processing (an arbitrary, JSON + serializable object), that allows you to determine whether the result is correct + or not. For example, in an app that generates SQL queries, the `output` should + be the _result_ of the SQL query generated by the model, not the query itself, + because there may be multiple valid queries that answer a single question"},"project_id":{"description":"Unique + identifier for the project that the experiment belongs under","format":"uuid","type":"string"},"root_span_id":{"description":"A + unique identifier for the trace this experiment event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"additionalProperties":{},"description":"Human-identifying + attributes of the span, such as name, type, etc.","properties":{"name":{"description":"Name + of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A + unique identifier used to link different experiment events together as part + of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) + for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}}},"realtime_state":{"type":"on","minimum_xact_id":"1000197642012143506","read_bytes":5773,"actual_xact_id":"1000197646831505354"},"freshness_state":{"last_processed_xact_id":"1000197642012143506","last_considered_xact_id":"1000197646831505354"},"warnings":[]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Fri, 07 Aug 2026 13:58:43 GMT + Via: + - 1.1 54704f25530eb7440881e3a166d68472.cloudfront.net (CloudFront), 1.1 cb0c6226aa19d81a39519501df383968.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - kalf9jIBac-nsn1YnzgplSKrD-uUIV7_r3aRoOavTCC9lAC24WLvCA== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a75e493-77ef24623d22e2bc6baaefb9;Parent=7bb41a3f340b34ff;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - private, no-cache + content-length: + - '7788' + vary: + - Origin + x-amz-apigw-id: + - BvCnGGgqIAMED6Q= + x-amzn-RequestId: + - fda7e21e-796a-4ddc-b50b-0f03d6ad2260 + x-bt-api-duration-ms: + - '356' + x-bt-brainstore-duration-ms: + - '291' + x-bt-internal-trace-id: + - 6a75e49300000000240071091f23e5fc + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/harbor/cassettes/0.16.0/test_atif_import_scopes_tool_calls_per_turn_and_reports_bounded_content.yaml b/py/src/braintrust/integrations/harbor/cassettes/0.16.0/test_atif_import_scopes_tool_calls_per_turn_and_reports_bounded_content.yaml new file mode 100644 index 00000000..65ed3485 --- /dev/null +++ b/py/src/braintrust/integrations/harbor/cassettes/0.16.0/test_atif_import_scopes_tool_calls_per_turn_and_reports_bounded_content.yaml @@ -0,0 +1,663 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/apikey/login + response: + body: + string: '{"org_info":[{"id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"Braintrust + SDKs","api_url":"https://api.braintrust.dev","git_metadata":{"collect":"some","fields":["commit","branch","tag","dirty","author_name","author_email","commit_message","commit_time"]},"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '376' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-OGEyY2U5OGItMGZhYy00ODRlLWFlM2EtODExZWQxOWU3OTA5'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 07 Aug 2026 13:58:43 GMT + Etag: + - '"13vsc5ye8flag"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Bt-Was-Udf-Cached: + - 'true' + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/apikey/login + X-Nonce: + - OGEyY2U5OGItMGZhYy00ODRlLWFlM2EtODExZWQxOWU3OTA5 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::cbfrg-1786111123906-b203b5295e98 + status: + code: 200 + message: OK +- request: + body: '{"project_name": "python-sdk-harbor-tests", "project_id": null, "org_id": + "5abfae3a-7aa7-4653-a9c8-b3efcb18f584", "update": true, "experiment_name": "harbor-atif-content-bounds", + "public": false}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '195' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/experiment/register + response: + body: + string: '{"project":{"id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-harbor-tests","description":null,"created":"2026-07-30T23:19:36.349Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null},"experiment":{"id":"5338e98e-ec96-47bd-8aab-99dc1aefcc74","project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","name":"harbor-atif-content-bounds","description":null,"created":"2026-08-06T17:31:42.204Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","metadata":null,"tags":null}}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '753' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-NzQyNzQ5ZjUtZDI3NC00NzM4LTliZWUtZTE0Y2Y2MDFiZTk2'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 07 Aug 2026 13:58:44 GMT + Etag: + - '"rgn3ryj073kx"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/experiment/register + X-Nonce: + - NzQyNzQ5ZjUtZDI3NC00NzM4LTliZWUtZTE0Y2Y2MDFiZTk2 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::zssg2-1786111124067-270930bc6242 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/version + response: + body: + string: '{"version":"2.9.0","date_version":"20260805","ff_version":44,"commit":"787a5ad8b0dd2090d79c37513ba5cc3d21758ddc","deployment_mode":"lambda","deployment_type":"custom","brainstore_default":"force","brainstore_can_contain_row_refs":true,"skip_pg_config":"all","has_realtime_wal_bucket":true,"brainstore_wal_footer_version":"v3","brainstore_wal_use_efficient_format":true,"has_logs2":true,"brainstore_export_enabled":true,"js":true,"universal":true,"code_execution":true,"logs3_payload_max_bytes":5242880,"control_plane_telemetry":["status","memprof","usage"]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 07 Aug 2026 13:58:44 GMT + Via: + - 1.1 54704f25530eb7440881e3a166d68472.cloudfront.net (CloudFront), 1.1 bc9d715161855640c4738aa7390d934e.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - BkB-sZmsXHtiVKpztSkknqiX0GyEZL91n7qjt2Z_zozKdGJhGZQf8Q== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a75e494-49713ef13cd4046460cecce9;Parent=3ac681fb7f0858ad;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '557' + etag: + - W/"22d-RtkjTszsylxX5VGRelXw9TnJwV8" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin + x-amz-apigw-id: + - BvCnQHhAoAMEgzQ= + x-amzn-Remapped-content-length: + - '557' + x-amzn-RequestId: + - 7b1344d5-a76d-4fbc-b838-ad65f7b61328 + x-bt-internal-trace-id: + - 6a75e494000000005b64cc9889257bc9 + status: + code: 200 + message: OK +- request: + body: '{"rows": [{"_is_merge": false, "context": {"caller_filename": "[REDACTED_PATH]", + "caller_functionname": "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": + {"instrumentation": {"name": "braintrust-python-logger"}, "name": "braintrust.sdk.python", + "version": "0.31.1"}}, "created": "2026-08-07T13:58:43.745999+00:00", "experiment_id": + "5338e98e-ec96-47bd-8aab-99dc1aefcc74", "id": "b4de2f31-7c05-5a9e-8d64-1f3a6c9b2e70", + "metrics": {"start": 1767225600.0}, "root_span_id": "fa9569252a62be16d8a26be3d5248bdb", + "span_attributes": {"exec_counter": 5, "name": "agent_execution", "type": "task"}, + "span_id": "b093a2d433b7e6e4", "span_parents": null}], "api_version": 2}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '831' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/logs3 + response: + body: + string: '{"ids":["b4de2f31-7c05-5a9e-8d64-1f3a6c9b2e70"],"xact_id":"1000197646831645862"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 07 Aug 2026 13:58:44 GMT + Via: + - 1.1 54704f25530eb7440881e3a166d68472.cloudfront.net (CloudFront), 1.1 41c02c3f5acef4f58284b65a8f7a983a.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - zSMMl59FQW9k2is3dmHuMqZbahSy_nRQjYBD9i9yYYkK6Xr6DfMXYw== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a75e494-0114a1dc6a01ebc01a23e932;Parent=288e91c79d35349a;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '80' + etag: + - W/"50-DAb7ckFwML3Ryn/O1/xufQNu1x4" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - BvCnSHzxoAMEtNw= + x-amzn-RequestId: + - cd61478e-3fa7-44dc-b79c-efa0f7998d91 + x-bt-internal-trace-id: + - 6a75e4940000000076a6da237cc3c82e + status: + code: 200 + message: OK +- request: + body: '{"rows": [{"context": {"caller_filename": "[REDACTED_PATH]", "caller_functionname": + "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": {"instrumentation": + {"name": "braintrust.plugin.harbor"}, "name": "braintrust.sdk.python", "version": + "0.31.1"}}, "created": "2026-08-07T13:58:43.747715+00:00", "experiment_id": + "5338e98e-ec96-47bd-8aab-99dc1aefcc74", "id": "1b018f6e-2160-5ba5-b292-0cbf2ebe1769", + "input": [{"content": "Read /app/answer.txt and then /app/notes.txt.", "role": + "user"}], "metadata": {"model": "gpt-4o-mini", "provider": "openai"}, "metrics": + {"completion_tokens": 5, "end": 1767225602.0, "prompt_tokens": 12, "start": + 1767225601.0, "tokens": 17}, "output": {"content": "Reading the answer file.", + "role": "assistant", "tool_calls": [{"function": {"arguments": "{\"path\": \"/app/answer.txt\"}", + "name": "read_file"}, "id": "call_1", "type": "function"}]}, "root_span_id": + "fa9569252a62be16d8a26be3d5248bdb", "span_attributes": {"exec_counter": 6, "name": + "chat.completions.create", "type": "llm"}, "span_id": "17e7cb2440b583b9", "span_parents": + ["b093a2d433b7e6e4"]},{"context": {"caller_filename": "[REDACTED_PATH]", "caller_functionname": + "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": {"instrumentation": + {"name": "braintrust.plugin.harbor"}, "name": "braintrust.sdk.python", "version": + "0.31.1"}}, "created": "2026-08-07T13:58:43.748392+00:00", "experiment_id": + "5338e98e-ec96-47bd-8aab-99dc1aefcc74", "id": "005dcf33-1429-5b69-8ac1-59e757a830cc", + "input": {"path": "/app/answer.txt"}, "metadata": {"tool_call_id": "call_1"}, + "metrics": {"end": 1767225602.0, "start": 1767225601.0}, "output": "42", "root_span_id": + "fa9569252a62be16d8a26be3d5248bdb", "span_attributes": {"exec_counter": 7, "name": + "read_file", "type": "tool"}, "span_id": "ac1994818737e95e", "span_parents": + ["b093a2d433b7e6e4"]},{"context": {"caller_filename": "[REDACTED_PATH]", "caller_functionname": + "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": {"instrumentation": + {"name": "braintrust.plugin.harbor"}, "name": "braintrust.sdk.python", "version": + "0.31.1"}}, "created": "2026-08-07T13:58:43.749305+00:00", "experiment_id": + "5338e98e-ec96-47bd-8aab-99dc1aefcc74", "id": "2b19b087-ab1a-5197-a052-14367c7a1bbe", + "input": [{"content": "Read /app/answer.txt and then /app/notes.txt.", "role": + "user"}, {"content": "Reading the answer file.", "role": "assistant", "tool_calls": + [{"function": {"arguments": "{\"path\": \"/app/answer.txt\"}", "name": "read_file"}, + "id": "call_1", "type": "function"}]}, {"content": "42", "role": "tool", "tool_call_id": + "call_1"}], "metadata": {"model": "gpt-4o-mini", "provider": "openai"}, "metrics": + {"completion_tokens": 6, "end": 1767225603.0, "prompt_tokens": 14, "start": + 1767225602.0, "tokens": 20}, "output": {"content": "Now the notes.", "role": + "assistant", "tool_calls": [{"function": {"arguments": "{\"path\": \"/app/notes.txt\"}", + "name": "read_file"}, "id": "call_1", "type": "function"}]}, "root_span_id": + "fa9569252a62be16d8a26be3d5248bdb", "span_attributes": {"exec_counter": 8, "name": + "chat.completions.create", "type": "llm"}, "span_id": "6d4b9fb4781d104e", "span_parents": + ["b093a2d433b7e6e4"]},{"context": {"caller_filename": "[REDACTED_PATH]", "caller_functionname": + "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": {"instrumentation": + {"name": "braintrust.plugin.harbor"}, "name": "braintrust.sdk.python", "version": + "0.31.1"}}, "created": "2026-08-07T13:58:43.750308+00:00", "experiment_id": + "5338e98e-ec96-47bd-8aab-99dc1aefcc74", "id": "31c21612-6f18-5ff7-9242-ee74839fe84b", + "input": {"path": "/app/notes.txt"}, "metadata": {"tool_call_id": "call_1"}, + "metrics": {"end": 1767225603.0, "start": 1767225602.0}, "output": "none", "root_span_id": + "fa9569252a62be16d8a26be3d5248bdb", "span_attributes": {"exec_counter": 9, "name": + "read_file", "type": "tool"}, "span_id": "08087920b69f7784", "span_parents": + ["b093a2d433b7e6e4"]},{"context": {"caller_filename": "[REDACTED_PATH]", "caller_functionname": + "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": {"instrumentation": + {"name": "braintrust.plugin.harbor"}, "name": "braintrust.sdk.python", "version": + "0.31.1"}}, "created": "2026-08-07T13:58:43.750794+00:00", "experiment_id": + "5338e98e-ec96-47bd-8aab-99dc1aefcc74", "id": "78469d79-2548-5756-8e18-50d6766f7362", + "input": {"source": "agent"}, "metrics": {"end": 1767225604.0, "start": 1767225603.0}, + "output": {"message": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy", + "tool_call_count": 0}, "root_span_id": "fa9569252a62be16d8a26be3d5248bdb", "span_attributes": + {"exec_counter": 10, "name": "trajectory.step.4", "type": "task"}, "span_id": + "0ba74e33109bd871", "span_parents": ["b093a2d433b7e6e4"]},{"_is_merge": true, + "experiment_id": "5338e98e-ec96-47bd-8aab-99dc1aefcc74", "id": "b4de2f31-7c05-5a9e-8d64-1f3a6c9b2e70", + "metrics": {"end": 1767225604.0}, "root_span_id": "fa9569252a62be16d8a26be3d5248bdb", + "span_id": "b093a2d433b7e6e4", "span_parents": null}], "api_version": 2}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '6042' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/logs3 + response: + body: + string: '{"ids":["1b018f6e-2160-5ba5-b292-0cbf2ebe1769","005dcf33-1429-5b69-8ac1-59e757a830cc","2b19b087-ab1a-5197-a052-14367c7a1bbe","31c21612-6f18-5ff7-9242-ee74839fe84b","78469d79-2548-5756-8e18-50d6766f7362","b4de2f31-7c05-5a9e-8d64-1f3a6c9b2e70"],"xact_id":"1000197646831713234"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 07 Aug 2026 13:58:45 GMT + Via: + - 1.1 e7e881849322d751aeeb9605914b08b4.cloudfront.net (CloudFront), 1.1 7293b56f3a0eb541aadcbcaa0146d528.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - KogwDKwl0Wtf7Jd_qJ8JNayl8mkY3DwZ0XMThJKUvAJwX9wXEIsC3Q== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a75e495-158de08b4a69487e333f7c7f;Parent=3d8c067c55d2e461;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '275' + etag: + - W/"113-dEQExPzN/NMCvnQXkHHCNNUNN9E" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - BvCnXGKFoAMEDZA= + x-amzn-RequestId: + - eac7b2f5-750c-45a8-9a7c-5475e2b72c41 + x-bt-internal-trace-id: + - 6a75e495000000005045589303de9a7f + status: + code: 200 + message: OK +- request: + body: '{"query": {"select": [{"op": "star"}], "from": {"op": "function", "name": + {"op": "ident", "name": ["experiment"]}, "args": [{"op": "literal", "value": + "5338e98e-ec96-47bd-8aab-99dc1aefcc74"}]}, "cursor": null, "limit": 1000}, "use_columnstore": + false, "brainstore_realtime": true, "query_source": "py_sdk_object_fetcher_experiment"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip + Connection: + - keep-alive + Content-Length: + - '332' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/btql + response: + body: + string: '{"data":[{"_pagination_key":"p07671288873121611780","_xact_id":"1000197646831713234","audit_data":[{"_xact_id":"1000197646831713234","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.1"}},"created":"2026-08-07T13:58:43.750Z","error":null,"expected":null,"experiment_id":"5338e98e-ec96-47bd-8aab-99dc1aefcc74","facets":null,"id":"78469d79-2548-5756-8e18-50d6766f7362","input":{"source":"agent"},"is_root":false,"metadata":null,"metrics":{"end":1767225604,"start":1767225603},"origin":null,"output":{"message":"yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy","tool_call_count":0},"project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"fa9569252a62be16d8a26be3d5248bdb","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":10,"name":"trajectory.step.4","type":"task"},"span_id":"0ba74e33109bd871","span_parents":["b093a2d433b7e6e4"],"tags":null},{"_pagination_key":"p07671288873121611779","_xact_id":"1000197646831713234","audit_data":[{"_xact_id":"1000197646831713234","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.1"}},"created":"2026-08-07T13:58:43.750Z","error":null,"expected":null,"experiment_id":"5338e98e-ec96-47bd-8aab-99dc1aefcc74","facets":null,"id":"31c21612-6f18-5ff7-9242-ee74839fe84b","input":{"path":"/app/notes.txt"},"is_root":false,"metadata":{"tool_call_id":"call_1"},"metrics":{"end":1767225603,"start":1767225602},"origin":null,"output":"none","project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"fa9569252a62be16d8a26be3d5248bdb","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":9,"name":"read_file","type":"tool"},"span_id":"08087920b69f7784","span_parents":["b093a2d433b7e6e4"],"tags":null},{"_pagination_key":"p07671288873121611778","_xact_id":"1000197646831713234","audit_data":[{"_xact_id":"1000197646831713234","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.1"}},"created":"2026-08-07T13:58:43.749Z","error":null,"expected":null,"experiment_id":"5338e98e-ec96-47bd-8aab-99dc1aefcc74","facets":null,"id":"2b19b087-ab1a-5197-a052-14367c7a1bbe","input":[{"content":"Read + /app/answer.txt and then /app/notes.txt.","role":"user"},{"content":"Reading + the answer file.","role":"assistant","tool_calls":[{"function":{"arguments":"{\"path\": + \"/app/answer.txt\"}","name":"read_file"},"id":"call_1","type":"function"}]},{"content":"42","role":"tool","tool_call_id":"call_1"}],"is_root":false,"metadata":{"model":"gpt-4o-mini","provider":"openai"},"metrics":{"completion_tokens":6,"end":1767225603,"prompt_tokens":14,"start":1767225602,"tokens":20},"origin":null,"output":{"content":"Now + the notes.","role":"assistant","tool_calls":[{"function":{"arguments":"{\"path\": + \"/app/notes.txt\"}","name":"read_file"},"id":"call_1","type":"function"}]},"project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"fa9569252a62be16d8a26be3d5248bdb","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":8,"name":"chat.completions.create","type":"llm"},"span_id":"6d4b9fb4781d104e","span_parents":["b093a2d433b7e6e4"],"tags":null},{"_pagination_key":"p07671288873121611777","_xact_id":"1000197646831713234","audit_data":[{"_xact_id":"1000197646831713234","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.1"}},"created":"2026-08-07T13:58:43.748Z","error":null,"expected":null,"experiment_id":"5338e98e-ec96-47bd-8aab-99dc1aefcc74","facets":null,"id":"005dcf33-1429-5b69-8ac1-59e757a830cc","input":{"path":"/app/answer.txt"},"is_root":false,"metadata":{"tool_call_id":"call_1"},"metrics":{"end":1767225602,"start":1767225601},"origin":null,"output":"42","project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"fa9569252a62be16d8a26be3d5248bdb","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":7,"name":"read_file","type":"tool"},"span_id":"ac1994818737e95e","span_parents":["b093a2d433b7e6e4"],"tags":null},{"_pagination_key":"p07671288873121611776","_xact_id":"1000197646831713234","audit_data":[{"_xact_id":"1000197646831713234","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.1"}},"created":"2026-08-07T13:58:43.747Z","error":null,"expected":null,"experiment_id":"5338e98e-ec96-47bd-8aab-99dc1aefcc74","facets":null,"id":"1b018f6e-2160-5ba5-b292-0cbf2ebe1769","input":[{"content":"Read + /app/answer.txt and then /app/notes.txt.","role":"user"}],"is_root":false,"metadata":{"model":"gpt-4o-mini","provider":"openai"},"metrics":{"completion_tokens":5,"end":1767225602,"prompt_tokens":12,"start":1767225601,"tokens":17},"origin":null,"output":{"content":"Reading + the answer file.","role":"assistant","tool_calls":[{"function":{"arguments":"{\"path\": + \"/app/answer.txt\"}","name":"read_file"},"id":"call_1","type":"function"}]},"project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"fa9569252a62be16d8a26be3d5248bdb","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":6,"name":"chat.completions.create","type":"llm"},"span_id":"17e7cb2440b583b9","span_parents":["b093a2d433b7e6e4"],"tags":null},{"_pagination_key":"p07671288868706320384","_xact_id":"1000197646831713234","audit_data":[{"_xact_id":"1000197646831645862","audit_data":{"action":"upsert"},"metadata":{},"source":"api"},{"_xact_id":"1000197646831713234","audit_data":{"action":"merge","from":null,"path":["metrics"],"to":{"end":1767225604}},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust-python-logger"},"name":"braintrust.sdk.python","version":"0.31.1"}},"created":"2026-08-07T13:58:43.745Z","error":null,"expected":null,"experiment_id":"5338e98e-ec96-47bd-8aab-99dc1aefcc74","facets":null,"id":"b4de2f31-7c05-5a9e-8d64-1f3a6c9b2e70","input":null,"is_root":true,"metadata":null,"metrics":{"end":1767225604,"start":1767225600},"origin":null,"output":null,"project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"fa9569252a62be16d8a26be3d5248bdb","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":5,"name":"agent_execution","type":"task"},"span_id":"b093a2d433b7e6e4","span_parents":null,"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"_pagination_key":{"description":"A + stable, time-ordered key that can be used to paginate over experiment events. + This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The + transaction id of an event is unique to the network operation that processed + the event insertion. Transaction ids are monotonically increasing over time + and can be used to retrieve a versioned snapshot of the experiment (see the + `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"additionalProperties":false,"properties":{"confidence":{"description":"Optional + confidence score for the classification","type":["number","null"]},"id":{"description":"Stable + classification identifier","type":"string"},"label":{"description":"Original + label of the classification item, which is useful for search and indexing + purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional + metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The + version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The + type of global function. Defaults to ''scorer''.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional + function identifier that produced the classification"}},"required":["id"],"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name + of the file in code where the experiment event was created","type":["string","null"]},"caller_functionname":{"description":"The + function in code which created the experiment event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The + timestamp the experiment event was created","format":"date-time","type":"string"},"error":{"description":"The + error that occurred, if any."},"expected":{"description":"The ground truth + value (an arbitrary, JSON serializable object) that you''d compare to `output` + to determine if your `output` value is correct or not. Braintrust currently + does not compare `output` to `expected` for you, since there are so many different + ways to do that correctly. Instead, these values are just used to help you + navigate your experiments while digging into analyses. However, we may later + use these values to re-score outputs or fine-tune your models"},"experiment_id":{"description":"Unique + identifier for the experiment","format":"uuid","type":"string"},"facets":{"anyOf":[{"additionalProperties":{"type":["string","null"]},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A + unique identifier for the experiment event. If you don''t provide one, Braintrust + will generate one for you","type":"string"},"input":{"description":"The arguments + that uniquely define a test case (an arbitrary, JSON serializable object). + Later on, Braintrust will use the `input` to know whether two test cases are + the same between experiments, so they should not contain experiment-specific + state. A simple rule of thumb is that if you run the same experiment twice, + the `input` should be identical"},"is_root":{"description":"Whether this span + is a root span","type":["boolean","null"]},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The + model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This + metric is deprecated"},"caller_functionname":{"description":"This metric is + deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A + unix timestamp recording when the section of code which produced the experiment + event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A + unix timestamp recording when the section of code which produced the experiment + event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"origin":{"anyOf":[{"description":"Reference + to the original object and event this was copied from.","properties":{"_xact_id":{"description":"Transaction + ID of the original event.","type":["string","null"]},"created":{"description":"Created + timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID + of the original event.","type":"string"},"object_id":{"description":"ID of + the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type + of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"required":["object_type","object_id","id"],"type":"object"},{"type":"null"}]},"output":{"description":"The + output of your application, including post-processing (an arbitrary, JSON + serializable object), that allows you to determine whether the result is correct + or not. For example, in an app that generates SQL queries, the `output` should + be the _result_ of the SQL query generated by the model, not the query itself, + because there may be multiple valid queries that answer a single question"},"project_id":{"description":"Unique + identifier for the project that the experiment belongs under","format":"uuid","type":"string"},"root_span_id":{"description":"A + unique identifier for the trace this experiment event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"additionalProperties":{},"description":"Human-identifying + attributes of the span, such as name, type, etc.","properties":{"name":{"description":"Name + of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A + unique identifier used to link different experiment events together as part + of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) + for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}}},"cursor":"anXklPSmAAA","realtime_state":{"type":"on","minimum_xact_id":"1000197642012355312","read_bytes":8418,"actual_xact_id":"1000197646831713234"},"freshness_state":{"last_processed_xact_id":"1000197642012355312","last_considered_xact_id":"1000197646831713234"},"warnings":[]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Fri, 07 Aug 2026 13:58:45 GMT + Via: + - 1.1 ba8b183ec1776a2582f8b77753f81c2a.cloudfront.net (CloudFront), 1.1 3340b5a392e45fce453c4d978abfd6be.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - MXHR1NhC1pVkWGqQX18pSVnU5Z8Vr8PtCkFJymsy5UEtGdqc1-FknA== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a75e495-3f205b7010067b5b67d08c97;Parent=75d68c5779a5ad38;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - private, no-cache + content-length: + - '17167' + vary: + - Origin + x-amz-apigw-id: + - BvCnaF6doAMEDxg= + x-amzn-RequestId: + - ffd48dd2-3521-4eb8-bf49-8ceb8a12dc52 + x-bt-api-duration-ms: + - '401' + x-bt-brainstore-duration-ms: + - '330' + x-bt-cursor: + - anXklPSmAAA + x-bt-internal-trace-id: + - 6a75e4950000000030add7ce6c843148 + status: + code: 200 + message: OK +- request: + body: '{"query": {"select": [{"op": "star"}], "from": {"op": "function", "name": + {"op": "ident", "name": ["experiment"]}, "args": [{"op": "literal", "value": + "5338e98e-ec96-47bd-8aab-99dc1aefcc74"}]}, "cursor": "anXklPSmAAA", "limit": + 1000}, "use_columnstore": false, "brainstore_realtime": true, "query_source": + "py_sdk_object_fetcher_experiment"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip + Connection: + - keep-alive + Content-Length: + - '341' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/btql + response: + body: + string: '{"data":[],"schema":{"type":"array","items":{"type":"object","properties":{"_pagination_key":{"description":"A + stable, time-ordered key that can be used to paginate over experiment events. + This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The + transaction id of an event is unique to the network operation that processed + the event insertion. Transaction ids are monotonically increasing over time + and can be used to retrieve a versioned snapshot of the experiment (see the + `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"additionalProperties":false,"properties":{"confidence":{"description":"Optional + confidence score for the classification","type":["number","null"]},"id":{"description":"Stable + classification identifier","type":"string"},"label":{"description":"Original + label of the classification item, which is useful for search and indexing + purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional + metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The + version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The + type of global function. Defaults to ''scorer''.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional + function identifier that produced the classification"}},"required":["id"],"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name + of the file in code where the experiment event was created","type":["string","null"]},"caller_functionname":{"description":"The + function in code which created the experiment event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The + timestamp the experiment event was created","format":"date-time","type":"string"},"error":{"description":"The + error that occurred, if any."},"expected":{"description":"The ground truth + value (an arbitrary, JSON serializable object) that you''d compare to `output` + to determine if your `output` value is correct or not. Braintrust currently + does not compare `output` to `expected` for you, since there are so many different + ways to do that correctly. Instead, these values are just used to help you + navigate your experiments while digging into analyses. However, we may later + use these values to re-score outputs or fine-tune your models"},"experiment_id":{"description":"Unique + identifier for the experiment","format":"uuid","type":"string"},"facets":{"anyOf":[{"additionalProperties":{"type":["string","null"]},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A + unique identifier for the experiment event. If you don''t provide one, Braintrust + will generate one for you","type":"string"},"input":{"description":"The arguments + that uniquely define a test case (an arbitrary, JSON serializable object). + Later on, Braintrust will use the `input` to know whether two test cases are + the same between experiments, so they should not contain experiment-specific + state. A simple rule of thumb is that if you run the same experiment twice, + the `input` should be identical"},"is_root":{"description":"Whether this span + is a root span","type":["boolean","null"]},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The + model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This + metric is deprecated"},"caller_functionname":{"description":"This metric is + deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A + unix timestamp recording when the section of code which produced the experiment + event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A + unix timestamp recording when the section of code which produced the experiment + event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"origin":{"anyOf":[{"description":"Reference + to the original object and event this was copied from.","properties":{"_xact_id":{"description":"Transaction + ID of the original event.","type":["string","null"]},"created":{"description":"Created + timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID + of the original event.","type":"string"},"object_id":{"description":"ID of + the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type + of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"required":["object_type","object_id","id"],"type":"object"},{"type":"null"}]},"output":{"description":"The + output of your application, including post-processing (an arbitrary, JSON + serializable object), that allows you to determine whether the result is correct + or not. For example, in an app that generates SQL queries, the `output` should + be the _result_ of the SQL query generated by the model, not the query itself, + because there may be multiple valid queries that answer a single question"},"project_id":{"description":"Unique + identifier for the project that the experiment belongs under","format":"uuid","type":"string"},"root_span_id":{"description":"A + unique identifier for the trace this experiment event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"additionalProperties":{},"description":"Human-identifying + attributes of the span, such as name, type, etc.","properties":{"name":{"description":"Name + of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A + unique identifier used to link different experiment events together as part + of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) + for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}}},"realtime_state":{"type":"on","minimum_xact_id":"1000197642012355312","read_bytes":8418,"actual_xact_id":"1000197646831713234"},"freshness_state":{"last_processed_xact_id":"1000197642012355312","last_considered_xact_id":"1000197646831713234"},"warnings":[]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Fri, 07 Aug 2026 13:58:46 GMT + Via: + - 1.1 54704f25530eb7440881e3a166d68472.cloudfront.net (CloudFront), 1.1 41c02c3f5acef4f58284b65a8f7a983a.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - nreBtgPnnlcwF_HYKKP1_n8nMYLMX_2-tjHp92zmKkfy0jg2eT0PQQ== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a75e495-372b3dba79e0ac783022fb0e;Parent=4704b087fe039ae2;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - private, no-cache + content-length: + - '7788' + vary: + - Origin + x-amz-apigw-id: + - BvCngEkUIAMEpMw= + x-amzn-RequestId: + - 8a933578-0294-4c09-ae46-23f27851e898 + x-bt-api-duration-ms: + - '145' + x-bt-brainstore-duration-ms: + - '80' + x-bt-internal-trace-id: + - 6a75e495000000007c755c099dde2e52 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/harbor/cassettes/latest/test_atif_import_round_trips_with_real_sdks.yaml b/py/src/braintrust/integrations/harbor/cassettes/latest/test_atif_import_round_trips_with_real_sdks.yaml new file mode 100644 index 00000000..89aed425 --- /dev/null +++ b/py/src/braintrust/integrations/harbor/cassettes/latest/test_atif_import_round_trips_with_real_sdks.yaml @@ -0,0 +1,643 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/apikey/login + response: + body: + string: '{"org_info":[{"id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"Braintrust + SDKs","api_url":"https://api.braintrust.dev","git_metadata":{"collect":"some","fields":["commit","branch","tag","dirty","author_name","author_email","commit_message","commit_time"]},"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '376' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-NDdiMTdhNzMtODc2Yy00Y2ZhLThmMTMtNDkwMDI4YmFmYzNh'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 06 Aug 2026 17:33:04 GMT + Etag: + - '"13vsc5ye8flag"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Bt-Was-Udf-Cached: + - 'true' + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/apikey/login + X-Nonce: + - NDdiMTdhNzMtODc2Yy00Y2ZhLThmMTMtNDkwMDI4YmFmYzNh + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::4trcb-1786037584800-30f79554e8d4 + status: + code: 200 + message: OK +- request: + body: '{"project_name": "python-sdk-harbor-tests", "project_id": null, "org_id": + "5abfae3a-7aa7-4653-a9c8-b3efcb18f584", "update": true, "experiment_name": "harbor-atif-import", + "public": false}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '187' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/experiment/register + response: + body: + string: '{"project":{"id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-harbor-tests","description":null,"created":"2026-07-30T23:19:36.349Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null},"experiment":{"id":"7aba8758-862e-41ef-9e77-6247c4b71287","project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","name":"harbor-atif-import","description":null,"created":"2026-07-30T23:19:36.349Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","metadata":null,"tags":null}}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '745' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-ODU3ZjNmYmItYjBhYi00Y2NmLWEyN2YtZmI2ZjE2ODA3Yzll'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 06 Aug 2026 17:33:05 GMT + Etag: + - '"29xhqzk7lxkp"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/experiment/register + X-Nonce: + - ODU3ZjNmYmItYjBhYi00Y2NmLWEyN2YtZmI2ZjE2ODA3Yzll + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::x5pqg-1786037585020-9ac23fe173f7 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/version + response: + body: + string: '{"version":"2.9.0","date_version":"20260805","ff_version":44,"commit":"787a5ad8b0dd2090d79c37513ba5cc3d21758ddc","deployment_mode":"lambda","deployment_type":"custom","brainstore_default":"force","brainstore_can_contain_row_refs":true,"skip_pg_config":"all","has_realtime_wal_bucket":true,"brainstore_wal_footer_version":"v3","brainstore_wal_use_efficient_format":true,"has_logs2":true,"brainstore_export_enabled":true,"js":true,"universal":true,"code_execution":true,"logs3_payload_max_bytes":5242880,"control_plane_telemetry":["status","memprof","usage"]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 06 Aug 2026 17:33:05 GMT + Via: + - 1.1 c998b153551d50824c68d959194353f6.cloudfront.net (CloudFront), 1.1 39d0b6c3836d173e719889fc86d67ce8.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - Ehw3gwb6fxzGNsZRaXZuzSFz61hZ6XxfEiOnRy0TlTGHDNT9uLpvOA== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a74c551-78d7eab1153aab12122311ac;Parent=277f99b04ac45872;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '557' + etag: + - W/"22d-RtkjTszsylxX5VGRelXw9TnJwV8" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin + x-amz-apigw-id: + - BsPExHU2IAMEnXA= + x-amzn-Remapped-content-length: + - '557' + x-amzn-RequestId: + - a0201b95-316a-45a3-aab8-54d41d0b364f + x-bt-internal-trace-id: + - 6a74c5510000000059226bdfb9edc420 + status: + code: 200 + message: OK +- request: + body: '{"rows": [{"_is_merge": false, "context": {"caller_filename": "[REDACTED_PATH]", + "caller_functionname": "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": + {"instrumentation": {"name": "braintrust-python-logger"}, "name": "braintrust.sdk.python", + "version": "0.31.1"}}, "created": "2026-08-06T17:33:04.602493+00:00", "experiment_id": + "7aba8758-862e-41ef-9e77-6247c4b71287", "id": "c7a87986-0192-5f40-9ac0-a535810f1fe7", + "metrics": {"start": 1767225600.0}, "root_span_id": "95696dbdeebe85bae5612938ab7f62f7", + "span_attributes": {"exec_counter": 1, "name": "agent_execution", "type": "task"}, + "span_id": "8daf37e077cea460", "span_parents": null}], "api_version": 2}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '831' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/logs3 + response: + body: + string: '{"ids":["c7a87986-0192-5f40-9ac0-a535810f1fe7"],"xact_id":"1000197642012141785"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 06 Aug 2026 17:33:05 GMT + Via: + - 1.1 c998b153551d50824c68d959194353f6.cloudfront.net (CloudFront), 1.1 777f4a7ed43b40353f84311869e119c8.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - KiT6pqFc7waTixuy95wF0xPsTQcbvkehRo2lxrlaYPqG_lN8lyqNWA== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a74c551-03a87b0f66fb60b4443e4801;Parent=4f9aa8aef58609ef;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '80' + etag: + - W/"50-7Cazvv0xkDAuZjmtvZSwPLB6CCA" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - BsPEzHMJoAMEcRg= + x-amzn-RequestId: + - b84b16a2-daa5-4e4c-b40d-9ad79cc2c9aa + x-bt-internal-trace-id: + - 6a74c551000000004e08e3120e4f5882 + status: + code: 200 + message: OK +- request: + body: '{"rows": [{"context": {"caller_filename": "[REDACTED_PATH]", "caller_functionname": + "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": {"instrumentation": + {"name": "braintrust.plugin.harbor"}, "name": "braintrust.sdk.python", "version": + "0.31.1"}}, "created": "2026-08-06T17:33:04.603121+00:00", "experiment_id": + "7aba8758-862e-41ef-9e77-6247c4b71287", "id": "7eb93a96-dfea-5318-bfff-63cf37ce89c6", + "input": [{"content": "What is 2+2?", "role": "user"}], "metadata": {"model": + "gpt-4o-mini", "provider": "openai", "tools": [{"function": {"name": "calculator", + "parameters": {"type": "object"}}, "type": "function"}]}, "metrics": {"completion_tokens": + 4, "end": 1767225602.0, "estimated_cost": 0.001, "prompt_tokens": 10, "start": + 1767225601.0, "tokens": 14}, "output": {"content": "I''ll calculate it.", "role": + "assistant", "tool_calls": [{"function": {"arguments": "{\"expression\": \"2+2\"}", + "name": "calculator"}, "id": "call_1", "type": "function"}]}, "root_span_id": + "95696dbdeebe85bae5612938ab7f62f7", "span_attributes": {"exec_counter": 2, "name": + "chat.completions.create", "type": "llm"}, "span_id": "fc1b62f81c0c0cef", "span_parents": + ["8daf37e077cea460"]},{"context": {"caller_filename": "[REDACTED_PATH]", "caller_functionname": + "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": {"instrumentation": + {"name": "braintrust.plugin.harbor"}, "name": "braintrust.sdk.python", "version": + "0.31.1"}}, "created": "2026-08-06T17:33:04.603434+00:00", "experiment_id": + "7aba8758-862e-41ef-9e77-6247c4b71287", "id": "b134c10f-7dc4-543a-9707-fa9af07dbbe8", + "input": {"expression": "2+2"}, "metadata": {"tool_call_id": "call_1"}, "metrics": + {"end": 1767225602.0, "start": 1767225601.0}, "output": "4", "root_span_id": + "95696dbdeebe85bae5612938ab7f62f7", "span_attributes": {"exec_counter": 3, "name": + "calculator", "type": "tool"}, "span_id": "d17a579f71773718", "span_parents": + ["8daf37e077cea460"]},{"context": {"caller_filename": "[REDACTED_PATH]", "caller_functionname": + "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": {"instrumentation": + {"name": "braintrust.plugin.harbor"}, "name": "braintrust.sdk.python", "version": + "0.31.1"}}, "created": "2026-08-06T17:33:04.604196+00:00", "experiment_id": + "7aba8758-862e-41ef-9e77-6247c4b71287", "id": "a2713e08-b2cb-5762-99d7-be29120f9880", + "input": [{"content": "What is 2+2?", "role": "user"}, {"content": "I''ll calculate + it.", "role": "assistant", "tool_calls": [{"function": {"arguments": "{\"expression\": + \"2+2\"}", "name": "calculator"}, "id": "call_1", "type": "function"}]}, {"content": + "4", "role": "tool", "tool_call_id": "call_1"}], "metadata": {"model": "gpt-4o-mini", + "provider": "openai", "tools": [{"function": {"name": "calculator", "parameters": + {"type": "object"}}, "type": "function"}]}, "metrics": {"completion_tokens": + 5, "end": 1767225603.0, "prompt_tokens": 15, "start": 1767225602.0, "tokens": + 20}, "output": {"content": "The answer is 4.", "role": "assistant"}, "root_span_id": + "95696dbdeebe85bae5612938ab7f62f7", "span_attributes": {"exec_counter": 4, "name": + "chat.completions.create", "type": "llm"}, "span_id": "15fa6276f0d0dbd7", "span_parents": + ["8daf37e077cea460"]},{"_is_merge": true, "experiment_id": "7aba8758-862e-41ef-9e77-6247c4b71287", + "id": "c7a87986-0192-5f40-9ac0-a535810f1fe7", "metrics": {"end": 1767225603.0}, + "root_span_id": "95696dbdeebe85bae5612938ab7f62f7", "span_id": "8daf37e077cea460", + "span_parents": null}], "api_version": 2}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '3935' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/logs3 + response: + body: + string: '{"ids":["7eb93a96-dfea-5318-bfff-63cf37ce89c6","b134c10f-7dc4-543a-9707-fa9af07dbbe8","a2713e08-b2cb-5762-99d7-be29120f9880","c7a87986-0192-5f40-9ac0-a535810f1fe7"],"xact_id":"1000197642012143506"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 06 Aug 2026 17:33:05 GMT + Via: + - 1.1 c998b153551d50824c68d959194353f6.cloudfront.net (CloudFront), 1.1 39d0b6c3836d173e719889fc86d67ce8.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - m-m_7xnORIxCoYyRGjqr-3-KsJp6umPLd8Ji7OC7bHyE7QnFZMLXWA== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a74c551-46b3b20c466d6e2866ae4f01;Parent=126e01bbf065d660;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '197' + etag: + - W/"c5-q6tyWX4dnU6xU/3YyqZjLWFJTy4" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - BsPE3H2IIAMEhsQ= + x-amzn-RequestId: + - 3e86d46a-b7a2-4ab9-b2a0-cc96dac7f0f5 + x-bt-internal-trace-id: + - 6a74c5510000000035c95d9bc394cc26 + status: + code: 200 + message: OK +- request: + body: '{"query": {"select": [{"op": "star"}], "from": {"op": "function", "name": + {"op": "ident", "name": ["experiment"]}, "args": [{"op": "literal", "value": + "7aba8758-862e-41ef-9e77-6247c4b71287"}]}, "cursor": null, "limit": 1000}, "use_columnstore": + false, "brainstore_realtime": true, "query_source": "py_sdk_object_fetcher_experiment"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip + Connection: + - keep-alive + Content-Length: + - '332' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/btql + response: + body: + string: '{"data":[{"_pagination_key":"p07670973017799917570","_xact_id":"1000197642012143506","audit_data":[{"_xact_id":"1000197642012143506","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.1"}},"created":"2026-08-06T17:33:04.604Z","error":null,"expected":null,"experiment_id":"7aba8758-862e-41ef-9e77-6247c4b71287","facets":null,"id":"a2713e08-b2cb-5762-99d7-be29120f9880","input":[{"content":"What + is 2+2?","role":"user"},{"content":"I''ll calculate it.","role":"assistant","tool_calls":[{"function":{"arguments":"{\"expression\": + \"2+2\"}","name":"calculator"},"id":"call_1","type":"function"}]},{"content":"4","role":"tool","tool_call_id":"call_1"}],"is_root":false,"metadata":{"model":"gpt-4o-mini","provider":"openai","tools":[{"function":{"name":"calculator","parameters":{"type":"object"}},"type":"function"}]},"metrics":{"completion_tokens":5,"end":1767225603,"prompt_tokens":15,"start":1767225602,"tokens":20},"origin":null,"output":{"content":"The + answer is 4.","role":"assistant"},"project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"95696dbdeebe85bae5612938ab7f62f7","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":4,"name":"chat.completions.create","type":"llm"},"span_id":"15fa6276f0d0dbd7","span_parents":["8daf37e077cea460"],"tags":null},{"_pagination_key":"p07670973017799917569","_xact_id":"1000197642012143506","audit_data":[{"_xact_id":"1000197642012143506","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.1"}},"created":"2026-08-06T17:33:04.603Z","error":null,"expected":null,"experiment_id":"7aba8758-862e-41ef-9e77-6247c4b71287","facets":null,"id":"b134c10f-7dc4-543a-9707-fa9af07dbbe8","input":{"expression":"2+2"},"is_root":false,"metadata":{"tool_call_id":"call_1"},"metrics":{"end":1767225602,"start":1767225601},"origin":null,"output":"4","project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"95696dbdeebe85bae5612938ab7f62f7","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":3,"name":"calculator","type":"tool"},"span_id":"d17a579f71773718","span_parents":["8daf37e077cea460"],"tags":null},{"_pagination_key":"p07670973017799917568","_xact_id":"1000197642012143506","audit_data":[{"_xact_id":"1000197642012143506","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.1"}},"created":"2026-08-06T17:33:04.603Z","error":null,"expected":null,"experiment_id":"7aba8758-862e-41ef-9e77-6247c4b71287","facets":null,"id":"7eb93a96-dfea-5318-bfff-63cf37ce89c6","input":[{"content":"What + is 2+2?","role":"user"}],"is_root":false,"metadata":{"model":"gpt-4o-mini","provider":"openai","tools":[{"function":{"name":"calculator","parameters":{"type":"object"}},"type":"function"}]},"metrics":{"completion_tokens":4,"end":1767225602,"estimated_cost":0.001,"prompt_tokens":10,"start":1767225601,"tokens":14},"origin":null,"output":{"content":"I''ll + calculate it.","role":"assistant","tool_calls":[{"function":{"arguments":"{\"expression\": + \"2+2\"}","name":"calculator"},"id":"call_1","type":"function"}]},"project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"95696dbdeebe85bae5612938ab7f62f7","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":2,"name":"chat.completions.create","type":"llm"},"span_id":"fc1b62f81c0c0cef","span_parents":["8daf37e077cea460"],"tags":null},{"_pagination_key":"p07670973017687130112","_xact_id":"1000197642012143506","audit_data":[{"_xact_id":"1000197642012141785","audit_data":{"action":"upsert"},"metadata":{},"source":"api"},{"_xact_id":"1000197642012143506","audit_data":{"action":"merge","from":null,"path":["metrics"],"to":{"end":1767225603}},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust-python-logger"},"name":"braintrust.sdk.python","version":"0.31.1"}},"created":"2026-08-06T17:33:04.602Z","error":null,"expected":null,"experiment_id":"7aba8758-862e-41ef-9e77-6247c4b71287","facets":null,"id":"c7a87986-0192-5f40-9ac0-a535810f1fe7","input":null,"is_root":true,"metadata":null,"metrics":{"end":1767225603,"start":1767225600},"origin":null,"output":null,"project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"95696dbdeebe85bae5612938ab7f62f7","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":1,"name":"agent_execution","type":"task"},"span_id":"8daf37e077cea460","span_parents":null,"tags":null},{"_pagination_key":"p07668465162241900545","_xact_id":"1000197603745304547","audit_data":[{"_xact_id":"1000197603738631632","audit_data":{"action":"upsert"},"metadata":{},"source":"api"},{"_xact_id":"1000197603742781733","audit_data":{"action":"upsert"},"metadata":{},"source":"api"},{"_xact_id":"1000197603745304547","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.0"}},"created":"2026-07-30T23:21:18.801Z","error":null,"expected":null,"experiment_id":"7aba8758-862e-41ef-9e77-6247c4b71287","facets":null,"id":"843c7c39-2f3a-5813-aa42-56da9fe5709f","input":{"expression":"2+2"},"is_root":false,"metadata":{"tool_call_id":"call_1"},"metrics":{"end":1767225602,"start":1767225601},"origin":null,"output":"4","project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"d43785839001054eaf1c4ac196c82112","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":3,"name":"calculator","type":"tool"},"span_id":"161550719b4ad9d1","span_parents":["2d261ed7ec06b709"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"_pagination_key":{"description":"A + stable, time-ordered key that can be used to paginate over experiment events. + This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The + transaction id of an event is unique to the network operation that processed + the event insertion. Transaction ids are monotonically increasing over time + and can be used to retrieve a versioned snapshot of the experiment (see the + `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"additionalProperties":false,"properties":{"confidence":{"description":"Optional + confidence score for the classification","type":["number","null"]},"id":{"description":"Stable + classification identifier","type":"string"},"label":{"description":"Original + label of the classification item, which is useful for search and indexing + purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional + metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The + version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The + type of global function. Defaults to ''scorer''.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional + function identifier that produced the classification"}},"required":["id"],"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name + of the file in code where the experiment event was created","type":["string","null"]},"caller_functionname":{"description":"The + function in code which created the experiment event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The + timestamp the experiment event was created","format":"date-time","type":"string"},"error":{"description":"The + error that occurred, if any."},"expected":{"description":"The ground truth + value (an arbitrary, JSON serializable object) that you''d compare to `output` + to determine if your `output` value is correct or not. Braintrust currently + does not compare `output` to `expected` for you, since there are so many different + ways to do that correctly. Instead, these values are just used to help you + navigate your experiments while digging into analyses. However, we may later + use these values to re-score outputs or fine-tune your models"},"experiment_id":{"description":"Unique + identifier for the experiment","format":"uuid","type":"string"},"facets":{"anyOf":[{"additionalProperties":{"type":["string","null"]},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A + unique identifier for the experiment event. If you don''t provide one, Braintrust + will generate one for you","type":"string"},"input":{"description":"The arguments + that uniquely define a test case (an arbitrary, JSON serializable object). + Later on, Braintrust will use the `input` to know whether two test cases are + the same between experiments, so they should not contain experiment-specific + state. A simple rule of thumb is that if you run the same experiment twice, + the `input` should be identical"},"is_root":{"description":"Whether this span + is a root span","type":["boolean","null"]},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The + model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This + metric is deprecated"},"caller_functionname":{"description":"This metric is + deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A + unix timestamp recording when the section of code which produced the experiment + event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A + unix timestamp recording when the section of code which produced the experiment + event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"origin":{"anyOf":[{"description":"Reference + to the original object and event this was copied from.","properties":{"_xact_id":{"description":"Transaction + ID of the original event.","type":["string","null"]},"created":{"description":"Created + timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID + of the original event.","type":"string"},"object_id":{"description":"ID of + the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type + of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"required":["object_type","object_id","id"],"type":"object"},{"type":"null"}]},"output":{"description":"The + output of your application, including post-processing (an arbitrary, JSON + serializable object), that allows you to determine whether the result is correct + or not. For example, in an app that generates SQL queries, the `output` should + be the _result_ of the SQL query generated by the model, not the query itself, + because there may be multiple valid queries that answer a single question"},"project_id":{"description":"Unique + identifier for the project that the experiment belongs under","format":"uuid","type":"string"},"root_span_id":{"description":"A + unique identifier for the trace this experiment event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"additionalProperties":{},"description":"Human-identifying + attributes of the span, such as name, type, etc.","properties":{"name":{"description":"Name + of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A + unique identifier used to link different experiment events together as part + of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) + for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}}},"cursor":"amvcb4/jAAE","realtime_state":{"type":"on","minimum_xact_id":"1000197642006594746","read_bytes":5773,"actual_xact_id":"1000197642012143506"},"freshness_state":{"last_processed_xact_id":"1000197642006594746","last_considered_xact_id":"1000197642012143506"},"warnings":[]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Thu, 06 Aug 2026 17:33:06 GMT + Via: + - 1.1 c998b153551d50824c68d959194353f6.cloudfront.net (CloudFront), 1.1 e6bfe249d47d39a52673337cf444c9ce.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - ZlYmt48njwlQEz0c2rBmuuAsMtxwi6L8nlHABeF3Nxup-_XmhtL1TQ== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a74c552-34b2718a4d9c910859bb9f66;Parent=16351249fb253cd2;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - private, no-cache + content-length: + - '15687' + vary: + - Origin + x-amz-apigw-id: + - BsPE6E34IAMEY8g= + x-amzn-RequestId: + - a69a70f2-0c04-4ca5-bc1f-8fcb49816a74 + x-bt-api-duration-ms: + - '326' + x-bt-brainstore-duration-ms: + - '284' + x-bt-cursor: + - amvcb4/jAAE + x-bt-internal-trace-id: + - 6a74c5520000000070ebb35ba91399e4 + status: + code: 200 + message: OK +- request: + body: '{"query": {"select": [{"op": "star"}], "from": {"op": "function", "name": + {"op": "ident", "name": ["experiment"]}, "args": [{"op": "literal", "value": + "7aba8758-862e-41ef-9e77-6247c4b71287"}]}, "cursor": "amvcb4/jAAE", "limit": + 1000}, "use_columnstore": false, "brainstore_realtime": true, "query_source": + "py_sdk_object_fetcher_experiment"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip + Connection: + - keep-alive + Content-Length: + - '341' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/btql + response: + body: + string: '{"data":[],"schema":{"type":"array","items":{"type":"object","properties":{"_pagination_key":{"description":"A + stable, time-ordered key that can be used to paginate over experiment events. + This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The + transaction id of an event is unique to the network operation that processed + the event insertion. Transaction ids are monotonically increasing over time + and can be used to retrieve a versioned snapshot of the experiment (see the + `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"additionalProperties":false,"properties":{"confidence":{"description":"Optional + confidence score for the classification","type":["number","null"]},"id":{"description":"Stable + classification identifier","type":"string"},"label":{"description":"Original + label of the classification item, which is useful for search and indexing + purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional + metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The + version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The + type of global function. Defaults to ''scorer''.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional + function identifier that produced the classification"}},"required":["id"],"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name + of the file in code where the experiment event was created","type":["string","null"]},"caller_functionname":{"description":"The + function in code which created the experiment event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The + timestamp the experiment event was created","format":"date-time","type":"string"},"error":{"description":"The + error that occurred, if any."},"expected":{"description":"The ground truth + value (an arbitrary, JSON serializable object) that you''d compare to `output` + to determine if your `output` value is correct or not. Braintrust currently + does not compare `output` to `expected` for you, since there are so many different + ways to do that correctly. Instead, these values are just used to help you + navigate your experiments while digging into analyses. However, we may later + use these values to re-score outputs or fine-tune your models"},"experiment_id":{"description":"Unique + identifier for the experiment","format":"uuid","type":"string"},"facets":{"anyOf":[{"additionalProperties":{"type":["string","null"]},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A + unique identifier for the experiment event. If you don''t provide one, Braintrust + will generate one for you","type":"string"},"input":{"description":"The arguments + that uniquely define a test case (an arbitrary, JSON serializable object). + Later on, Braintrust will use the `input` to know whether two test cases are + the same between experiments, so they should not contain experiment-specific + state. A simple rule of thumb is that if you run the same experiment twice, + the `input` should be identical"},"is_root":{"description":"Whether this span + is a root span","type":["boolean","null"]},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The + model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This + metric is deprecated"},"caller_functionname":{"description":"This metric is + deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A + unix timestamp recording when the section of code which produced the experiment + event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A + unix timestamp recording when the section of code which produced the experiment + event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"origin":{"anyOf":[{"description":"Reference + to the original object and event this was copied from.","properties":{"_xact_id":{"description":"Transaction + ID of the original event.","type":["string","null"]},"created":{"description":"Created + timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID + of the original event.","type":"string"},"object_id":{"description":"ID of + the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type + of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"required":["object_type","object_id","id"],"type":"object"},{"type":"null"}]},"output":{"description":"The + output of your application, including post-processing (an arbitrary, JSON + serializable object), that allows you to determine whether the result is correct + or not. For example, in an app that generates SQL queries, the `output` should + be the _result_ of the SQL query generated by the model, not the query itself, + because there may be multiple valid queries that answer a single question"},"project_id":{"description":"Unique + identifier for the project that the experiment belongs under","format":"uuid","type":"string"},"root_span_id":{"description":"A + unique identifier for the trace this experiment event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"additionalProperties":{},"description":"Human-identifying + attributes of the span, such as name, type, etc.","properties":{"name":{"description":"Name + of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A + unique identifier used to link different experiment events together as part + of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) + for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}}},"realtime_state":{"type":"on","minimum_xact_id":"1000197642006594746","read_bytes":5773,"actual_xact_id":"1000197642012143506"},"freshness_state":{"last_processed_xact_id":"1000197642006594746","last_considered_xact_id":"1000197642012143506"},"warnings":[]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Thu, 06 Aug 2026 17:33:06 GMT + Via: + - 1.1 e28f812a5672adfe3f02c5f108acee8c.cloudfront.net (CloudFront), 1.1 b734db9b28028c2ed717c3d72b3b45b8.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - msr45EJuNpojNhzQA1spHiZdtWmHKgB3efWObo-SdixGQppespLsDA== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a74c552-01c6fa4f67652a885fd7aac2;Parent=47ac4d8328289cda;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - private, no-cache + content-length: + - '7788' + vary: + - Origin + x-amz-apigw-id: + - BsPFAH5goAMEOYQ= + x-amzn-RequestId: + - 8476b61e-29dd-436e-89d0-706d14353dda + x-bt-api-duration-ms: + - '141' + x-bt-brainstore-duration-ms: + - '95' + x-bt-internal-trace-id: + - 6a74c55200000000644fc50343023d6a + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/harbor/cassettes/latest/test_atif_import_scopes_tool_calls_per_turn_and_reports_bounded_content.yaml b/py/src/braintrust/integrations/harbor/cassettes/latest/test_atif_import_scopes_tool_calls_per_turn_and_reports_bounded_content.yaml new file mode 100644 index 00000000..39222631 --- /dev/null +++ b/py/src/braintrust/integrations/harbor/cassettes/latest/test_atif_import_scopes_tool_calls_per_turn_and_reports_bounded_content.yaml @@ -0,0 +1,663 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/apikey/login + response: + body: + string: '{"org_info":[{"id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"Braintrust + SDKs","api_url":"https://api.braintrust.dev","git_metadata":{"collect":"some","fields":["commit","branch","tag","dirty","author_name","author_email","commit_message","commit_time"]},"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '376' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-MmRmYjE3NDktMDI5YS00OWE4LThkNGYtZWFhNmE3ZDdjNDll'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 06 Aug 2026 17:33:07 GMT + Etag: + - '"13vsc5ye8flag"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Bt-Was-Udf-Cached: + - 'true' + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/apikey/login + X-Nonce: + - MmRmYjE3NDktMDI5YS00OWE4LThkNGYtZWFhNmE3ZDdjNDll + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::pmb9k-1786037587071-3708f1b59e89 + status: + code: 200 + message: OK +- request: + body: '{"project_name": "python-sdk-harbor-tests", "project_id": null, "org_id": + "5abfae3a-7aa7-4653-a9c8-b3efcb18f584", "update": true, "experiment_name": "harbor-atif-content-bounds", + "public": false}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '195' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/experiment/register + response: + body: + string: '{"project":{"id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-harbor-tests","description":null,"created":"2026-07-30T23:19:36.349Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null},"experiment":{"id":"5338e98e-ec96-47bd-8aab-99dc1aefcc74","project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","name":"harbor-atif-content-bounds","description":null,"created":"2026-08-06T17:31:42.204Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","metadata":null,"tags":null}}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '753' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-YjRjMmJiMTUtMDgzYS00YWRiLWJhMDItNDdlNjdjODVkMzYx'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 06 Aug 2026 17:33:07 GMT + Etag: + - '"rgn3ryj073kx"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/experiment/register + X-Nonce: + - YjRjMmJiMTUtMDgzYS00YWRiLWJhMDItNDdlNjdjODVkMzYx + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::jr9hg-1786037587306-f1532acec786 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/version + response: + body: + string: '{"version":"2.9.0","date_version":"20260805","ff_version":44,"commit":"787a5ad8b0dd2090d79c37513ba5cc3d21758ddc","deployment_mode":"lambda","deployment_type":"custom","brainstore_default":"force","brainstore_can_contain_row_refs":true,"skip_pg_config":"all","has_realtime_wal_bucket":true,"brainstore_wal_footer_version":"v3","brainstore_wal_use_efficient_format":true,"has_logs2":true,"brainstore_export_enabled":true,"js":true,"universal":true,"code_execution":true,"logs3_payload_max_bytes":5242880,"control_plane_telemetry":["status","memprof","usage"]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 06 Aug 2026 17:33:07 GMT + Via: + - 1.1 8250156022879efefd7a589c8ba8c706.cloudfront.net (CloudFront), 1.1 50d743941b822ae5fa30db69233863a6.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - fXwKD6mRuDnkal9uINMEswnCjtkdfclRiNROMfxHExBh5_on14xe9Q== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a74c553-2190a7e27780b94a5b163a2b;Parent=431bde0b02ee0460;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '557' + etag: + - W/"22d-RtkjTszsylxX5VGRelXw9TnJwV8" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin + x-amz-apigw-id: + - BsPFIF88oAMEejA= + x-amzn-Remapped-content-length: + - '557' + x-amzn-RequestId: + - ad42115c-dacb-46fa-b0cf-96affe839e65 + x-bt-internal-trace-id: + - 6a74c553000000000c17eda35d08fb94 + status: + code: 200 + message: OK +- request: + body: '{"rows": [{"_is_merge": false, "context": {"caller_filename": "[REDACTED_PATH]", + "caller_functionname": "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": + {"instrumentation": {"name": "braintrust-python-logger"}, "name": "braintrust.sdk.python", + "version": "0.31.1"}}, "created": "2026-08-06T17:33:06.849575+00:00", "experiment_id": + "5338e98e-ec96-47bd-8aab-99dc1aefcc74", "id": "b4de2f31-7c05-5a9e-8d64-1f3a6c9b2e70", + "metrics": {"start": 1767225600.0}, "root_span_id": "7faa024933ef8ab006f3066e7e1da5df", + "span_attributes": {"exec_counter": 5, "name": "agent_execution", "type": "task"}, + "span_id": "204ece418cc67edd", "span_parents": null}], "api_version": 2}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '831' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/logs3 + response: + body: + string: '{"ids":["b4de2f31-7c05-5a9e-8d64-1f3a6c9b2e70"],"xact_id":"1000197642012288097"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 06 Aug 2026 17:33:07 GMT + Via: + - 1.1 e28f812a5672adfe3f02c5f108acee8c.cloudfront.net (CloudFront), 1.1 8e6145785e47042f882be946f6c05880.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - G3k7TuwGxsKU_VVXPlZSoTtb8TQNW8a3SZruySDTpOII95cXQxismQ== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a74c553-6658aa230757a1f264f6722a;Parent=1553af4395b6bae6;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '80' + etag: + - W/"50-jJaJ/bUIjFr8zQeDLIXre6ViORY" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - BsPFLE5EIAMEIEg= + x-amzn-RequestId: + - d37bfdcb-6e08-4c49-8751-cd60023231f8 + x-bt-internal-trace-id: + - 6a74c553000000002ca20ce7230cb439 + status: + code: 200 + message: OK +- request: + body: '{"rows": [{"context": {"caller_filename": "[REDACTED_PATH]", "caller_functionname": + "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": {"instrumentation": + {"name": "braintrust.plugin.harbor"}, "name": "braintrust.sdk.python", "version": + "0.31.1"}}, "created": "2026-08-06T17:33:06.851320+00:00", "experiment_id": + "5338e98e-ec96-47bd-8aab-99dc1aefcc74", "id": "1b018f6e-2160-5ba5-b292-0cbf2ebe1769", + "input": [{"content": "Read /app/answer.txt and then /app/notes.txt.", "role": + "user"}], "metadata": {"model": "gpt-4o-mini", "provider": "openai"}, "metrics": + {"completion_tokens": 5, "end": 1767225602.0, "prompt_tokens": 12, "start": + 1767225601.0, "tokens": 17}, "output": {"content": "Reading the answer file.", + "role": "assistant", "tool_calls": [{"function": {"arguments": "{\"path\": \"/app/answer.txt\"}", + "name": "read_file"}, "id": "call_1", "type": "function"}]}, "root_span_id": + "7faa024933ef8ab006f3066e7e1da5df", "span_attributes": {"exec_counter": 6, "name": + "chat.completions.create", "type": "llm"}, "span_id": "bacc529184c9cd42", "span_parents": + ["204ece418cc67edd"]},{"context": {"caller_filename": "[REDACTED_PATH]", "caller_functionname": + "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": {"instrumentation": + {"name": "braintrust.plugin.harbor"}, "name": "braintrust.sdk.python", "version": + "0.31.1"}}, "created": "2026-08-06T17:33:06.852738+00:00", "experiment_id": + "5338e98e-ec96-47bd-8aab-99dc1aefcc74", "id": "005dcf33-1429-5b69-8ac1-59e757a830cc", + "input": {"path": "/app/answer.txt"}, "metadata": {"tool_call_id": "call_1"}, + "metrics": {"end": 1767225602.0, "start": 1767225601.0}, "output": "42", "root_span_id": + "7faa024933ef8ab006f3066e7e1da5df", "span_attributes": {"exec_counter": 7, "name": + "read_file", "type": "tool"}, "span_id": "8803342d333bf93a", "span_parents": + ["204ece418cc67edd"]},{"context": {"caller_filename": "[REDACTED_PATH]", "caller_functionname": + "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": {"instrumentation": + {"name": "braintrust.plugin.harbor"}, "name": "braintrust.sdk.python", "version": + "0.31.1"}}, "created": "2026-08-06T17:33:06.853504+00:00", "experiment_id": + "5338e98e-ec96-47bd-8aab-99dc1aefcc74", "id": "2b19b087-ab1a-5197-a052-14367c7a1bbe", + "input": [{"content": "Read /app/answer.txt and then /app/notes.txt.", "role": + "user"}, {"content": "Reading the answer file.", "role": "assistant", "tool_calls": + [{"function": {"arguments": "{\"path\": \"/app/answer.txt\"}", "name": "read_file"}, + "id": "call_1", "type": "function"}]}, {"content": "42", "role": "tool", "tool_call_id": + "call_1"}], "metadata": {"model": "gpt-4o-mini", "provider": "openai"}, "metrics": + {"completion_tokens": 6, "end": 1767225603.0, "prompt_tokens": 14, "start": + 1767225602.0, "tokens": 20}, "output": {"content": "Now the notes.", "role": + "assistant", "tool_calls": [{"function": {"arguments": "{\"path\": \"/app/notes.txt\"}", + "name": "read_file"}, "id": "call_1", "type": "function"}]}, "root_span_id": + "7faa024933ef8ab006f3066e7e1da5df", "span_attributes": {"exec_counter": 8, "name": + "chat.completions.create", "type": "llm"}, "span_id": "44b18b9c8c5918bd", "span_parents": + ["204ece418cc67edd"]},{"context": {"caller_filename": "[REDACTED_PATH]", "caller_functionname": + "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": {"instrumentation": + {"name": "braintrust.plugin.harbor"}, "name": "braintrust.sdk.python", "version": + "0.31.1"}}, "created": "2026-08-06T17:33:06.854384+00:00", "experiment_id": + "5338e98e-ec96-47bd-8aab-99dc1aefcc74", "id": "31c21612-6f18-5ff7-9242-ee74839fe84b", + "input": {"path": "/app/notes.txt"}, "metadata": {"tool_call_id": "call_1"}, + "metrics": {"end": 1767225603.0, "start": 1767225602.0}, "output": "none", "root_span_id": + "7faa024933ef8ab006f3066e7e1da5df", "span_attributes": {"exec_counter": 9, "name": + "read_file", "type": "tool"}, "span_id": "f29b41c6cbea6ef9", "span_parents": + ["204ece418cc67edd"]},{"context": {"caller_filename": "[REDACTED_PATH]", "caller_functionname": + "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": {"instrumentation": + {"name": "braintrust.plugin.harbor"}, "name": "braintrust.sdk.python", "version": + "0.31.1"}}, "created": "2026-08-06T17:33:06.854908+00:00", "experiment_id": + "5338e98e-ec96-47bd-8aab-99dc1aefcc74", "id": "78469d79-2548-5756-8e18-50d6766f7362", + "input": {"source": "agent"}, "metrics": {"end": 1767225604.0, "start": 1767225603.0}, + "output": {"message": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy", + "tool_call_count": 0}, "root_span_id": "7faa024933ef8ab006f3066e7e1da5df", "span_attributes": + {"exec_counter": 10, "name": "trajectory.step.4", "type": "task"}, "span_id": + "43c1501c725297c7", "span_parents": ["204ece418cc67edd"]},{"_is_merge": true, + "experiment_id": "5338e98e-ec96-47bd-8aab-99dc1aefcc74", "id": "b4de2f31-7c05-5a9e-8d64-1f3a6c9b2e70", + "metrics": {"end": 1767225604.0}, "root_span_id": "7faa024933ef8ab006f3066e7e1da5df", + "span_id": "204ece418cc67edd", "span_parents": null}], "api_version": 2}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '6042' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/logs3 + response: + body: + string: '{"ids":["1b018f6e-2160-5ba5-b292-0cbf2ebe1769","005dcf33-1429-5b69-8ac1-59e757a830cc","2b19b087-ab1a-5197-a052-14367c7a1bbe","31c21612-6f18-5ff7-9242-ee74839fe84b","78469d79-2548-5756-8e18-50d6766f7362","b4de2f31-7c05-5a9e-8d64-1f3a6c9b2e70"],"xact_id":"1000197642012355312"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 06 Aug 2026 17:33:08 GMT + Via: + - 1.1 7b609f6f2da1597a3efb21b332d7ce54.cloudfront.net (CloudFront), 1.1 67dd4d73b80aece69a8e725c6d612b6e.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - 0zHB8FPOuH26nIhMiKJEJbBIJjf-n88oc4hNqGeq1fLsUYnEs9bk8A== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a74c554-2e79fb9f44f4aed30cdc4946;Parent=4d9f17ca7e9ea967;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '275' + etag: + - W/"113-tX9jx2cGctqk1GPZcmA/QdFuibU" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - BsPFNGVhoAMEm0w= + x-amzn-RequestId: + - 217a7c81-3464-4d47-a9bf-a8fa12f4b82b + x-bt-internal-trace-id: + - 6a74c55400000000734037ab91a35493 + status: + code: 200 + message: OK +- request: + body: '{"query": {"select": [{"op": "star"}], "from": {"op": "function", "name": + {"op": "ident", "name": ["experiment"]}, "args": [{"op": "literal", "value": + "5338e98e-ec96-47bd-8aab-99dc1aefcc74"}]}, "cursor": null, "limit": 1000}, "use_columnstore": + false, "brainstore_realtime": true, "query_source": "py_sdk_object_fetcher_experiment"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip + Connection: + - keep-alive + Content-Length: + - '332' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/btql + response: + body: + string: '{"data":[{"_pagination_key":"p07670973031680835588","_xact_id":"1000197642012355312","audit_data":[{"_xact_id":"1000197642012355312","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.1"}},"created":"2026-08-06T17:33:06.854Z","error":null,"expected":null,"experiment_id":"5338e98e-ec96-47bd-8aab-99dc1aefcc74","facets":null,"id":"78469d79-2548-5756-8e18-50d6766f7362","input":{"source":"agent"},"is_root":false,"metadata":null,"metrics":{"end":1767225604,"start":1767225603},"origin":null,"output":{"message":"yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy","tool_call_count":0},"project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"7faa024933ef8ab006f3066e7e1da5df","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":10,"name":"trajectory.step.4","type":"task"},"span_id":"43c1501c725297c7","span_parents":["204ece418cc67edd"],"tags":null},{"_pagination_key":"p07670973031680835587","_xact_id":"1000197642012355312","audit_data":[{"_xact_id":"1000197642012355312","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.1"}},"created":"2026-08-06T17:33:06.854Z","error":null,"expected":null,"experiment_id":"5338e98e-ec96-47bd-8aab-99dc1aefcc74","facets":null,"id":"31c21612-6f18-5ff7-9242-ee74839fe84b","input":{"path":"/app/notes.txt"},"is_root":false,"metadata":{"tool_call_id":"call_1"},"metrics":{"end":1767225603,"start":1767225602},"origin":null,"output":"none","project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"7faa024933ef8ab006f3066e7e1da5df","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":9,"name":"read_file","type":"tool"},"span_id":"f29b41c6cbea6ef9","span_parents":["204ece418cc67edd"],"tags":null},{"_pagination_key":"p07670973031680835586","_xact_id":"1000197642012355312","audit_data":[{"_xact_id":"1000197642012355312","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.1"}},"created":"2026-08-06T17:33:06.853Z","error":null,"expected":null,"experiment_id":"5338e98e-ec96-47bd-8aab-99dc1aefcc74","facets":null,"id":"2b19b087-ab1a-5197-a052-14367c7a1bbe","input":[{"content":"Read + /app/answer.txt and then /app/notes.txt.","role":"user"},{"content":"Reading + the answer file.","role":"assistant","tool_calls":[{"function":{"arguments":"{\"path\": + \"/app/answer.txt\"}","name":"read_file"},"id":"call_1","type":"function"}]},{"content":"42","role":"tool","tool_call_id":"call_1"}],"is_root":false,"metadata":{"model":"gpt-4o-mini","provider":"openai"},"metrics":{"completion_tokens":6,"end":1767225603,"prompt_tokens":14,"start":1767225602,"tokens":20},"origin":null,"output":{"content":"Now + the notes.","role":"assistant","tool_calls":[{"function":{"arguments":"{\"path\": + \"/app/notes.txt\"}","name":"read_file"},"id":"call_1","type":"function"}]},"project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"7faa024933ef8ab006f3066e7e1da5df","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":8,"name":"chat.completions.create","type":"llm"},"span_id":"44b18b9c8c5918bd","span_parents":["204ece418cc67edd"],"tags":null},{"_pagination_key":"p07670973031680835585","_xact_id":"1000197642012355312","audit_data":[{"_xact_id":"1000197642012355312","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.1"}},"created":"2026-08-06T17:33:06.852Z","error":null,"expected":null,"experiment_id":"5338e98e-ec96-47bd-8aab-99dc1aefcc74","facets":null,"id":"005dcf33-1429-5b69-8ac1-59e757a830cc","input":{"path":"/app/answer.txt"},"is_root":false,"metadata":{"tool_call_id":"call_1"},"metrics":{"end":1767225602,"start":1767225601},"origin":null,"output":"42","project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"7faa024933ef8ab006f3066e7e1da5df","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":7,"name":"read_file","type":"tool"},"span_id":"8803342d333bf93a","span_parents":["204ece418cc67edd"],"tags":null},{"_pagination_key":"p07670973031680835584","_xact_id":"1000197642012355312","audit_data":[{"_xact_id":"1000197642012355312","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.1"}},"created":"2026-08-06T17:33:06.851Z","error":null,"expected":null,"experiment_id":"5338e98e-ec96-47bd-8aab-99dc1aefcc74","facets":null,"id":"1b018f6e-2160-5ba5-b292-0cbf2ebe1769","input":[{"content":"Read + /app/answer.txt and then /app/notes.txt.","role":"user"}],"is_root":false,"metadata":{"model":"gpt-4o-mini","provider":"openai"},"metrics":{"completion_tokens":5,"end":1767225602,"prompt_tokens":12,"start":1767225601,"tokens":17},"origin":null,"output":{"content":"Reading + the answer file.","role":"assistant","tool_calls":[{"function":{"arguments":"{\"path\": + \"/app/answer.txt\"}","name":"read_file"},"id":"call_1","type":"function"}]},"project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"7faa024933ef8ab006f3066e7e1da5df","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":6,"name":"chat.completions.create","type":"llm"},"span_id":"bacc529184c9cd42","span_parents":["204ece418cc67edd"],"tags":null},{"_pagination_key":"p07670973027275833344","_xact_id":"1000197642012355312","audit_data":[{"_xact_id":"1000197642012288097","audit_data":{"action":"upsert"},"metadata":{},"source":"api"},{"_xact_id":"1000197642012355312","audit_data":{"action":"merge","from":null,"path":["metrics"],"to":{"end":1767225604}},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust-python-logger"},"name":"braintrust.sdk.python","version":"0.31.1"}},"created":"2026-08-06T17:33:06.849Z","error":null,"expected":null,"experiment_id":"5338e98e-ec96-47bd-8aab-99dc1aefcc74","facets":null,"id":"b4de2f31-7c05-5a9e-8d64-1f3a6c9b2e70","input":null,"is_root":true,"metadata":null,"metrics":{"end":1767225604,"start":1767225600},"origin":null,"output":null,"project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"7faa024933ef8ab006f3066e7e1da5df","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":5,"name":"agent_execution","type":"task"},"span_id":"204ece418cc67edd","span_parents":null,"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"_pagination_key":{"description":"A + stable, time-ordered key that can be used to paginate over experiment events. + This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The + transaction id of an event is unique to the network operation that processed + the event insertion. Transaction ids are monotonically increasing over time + and can be used to retrieve a versioned snapshot of the experiment (see the + `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"additionalProperties":false,"properties":{"confidence":{"description":"Optional + confidence score for the classification","type":["number","null"]},"id":{"description":"Stable + classification identifier","type":"string"},"label":{"description":"Original + label of the classification item, which is useful for search and indexing + purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional + metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The + version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The + type of global function. Defaults to ''scorer''.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional + function identifier that produced the classification"}},"required":["id"],"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name + of the file in code where the experiment event was created","type":["string","null"]},"caller_functionname":{"description":"The + function in code which created the experiment event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The + timestamp the experiment event was created","format":"date-time","type":"string"},"error":{"description":"The + error that occurred, if any."},"expected":{"description":"The ground truth + value (an arbitrary, JSON serializable object) that you''d compare to `output` + to determine if your `output` value is correct or not. Braintrust currently + does not compare `output` to `expected` for you, since there are so many different + ways to do that correctly. Instead, these values are just used to help you + navigate your experiments while digging into analyses. However, we may later + use these values to re-score outputs or fine-tune your models"},"experiment_id":{"description":"Unique + identifier for the experiment","format":"uuid","type":"string"},"facets":{"anyOf":[{"additionalProperties":{"type":["string","null"]},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A + unique identifier for the experiment event. If you don''t provide one, Braintrust + will generate one for you","type":"string"},"input":{"description":"The arguments + that uniquely define a test case (an arbitrary, JSON serializable object). + Later on, Braintrust will use the `input` to know whether two test cases are + the same between experiments, so they should not contain experiment-specific + state. A simple rule of thumb is that if you run the same experiment twice, + the `input` should be identical"},"is_root":{"description":"Whether this span + is a root span","type":["boolean","null"]},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The + model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This + metric is deprecated"},"caller_functionname":{"description":"This metric is + deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A + unix timestamp recording when the section of code which produced the experiment + event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A + unix timestamp recording when the section of code which produced the experiment + event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"origin":{"anyOf":[{"description":"Reference + to the original object and event this was copied from.","properties":{"_xact_id":{"description":"Transaction + ID of the original event.","type":["string","null"]},"created":{"description":"Created + timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID + of the original event.","type":"string"},"object_id":{"description":"ID of + the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type + of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"required":["object_type","object_id","id"],"type":"object"},{"type":"null"}]},"output":{"description":"The + output of your application, including post-processing (an arbitrary, JSON + serializable object), that allows you to determine whether the result is correct + or not. For example, in an app that generates SQL queries, the `output` should + be the _result_ of the SQL query generated by the model, not the query itself, + because there may be multiple valid queries that answer a single question"},"project_id":{"description":"Unique + identifier for the project that the experiment belongs under","format":"uuid","type":"string"},"root_span_id":{"description":"A + unique identifier for the trace this experiment event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"additionalProperties":{},"description":"Human-identifying + attributes of the span, such as name, type, etc.","properties":{"name":{"description":"Name + of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A + unique identifier used to link different experiment events together as part + of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) + for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}}},"cursor":"anTFU2RhAAA","realtime_state":{"type":"on","minimum_xact_id":"1000197642006809586","read_bytes":8418,"actual_xact_id":"1000197642012355312"},"freshness_state":{"last_processed_xact_id":"1000197642006809586","last_considered_xact_id":"1000197642012355312"},"warnings":[]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Thu, 06 Aug 2026 17:33:08 GMT + Via: + - 1.1 8250156022879efefd7a589c8ba8c706.cloudfront.net (CloudFront), 1.1 50d743941b822ae5fa30db69233863a6.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - -XJ5fTJPDVcOzOGlyaI_TZzo_EAbKiP3Y8KvvxnkizmNYBYuaxbFMQ== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a74c554-113829fa14754f0d26d8dacf;Parent=74e222d9ac598de4;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - private, no-cache + content-length: + - '17167' + vary: + - Origin + x-amz-apigw-id: + - BsPFQFNpoAMEOqQ= + x-amzn-RequestId: + - 965ab809-ca4d-418c-a65b-0ff03f149a7e + x-bt-api-duration-ms: + - '444' + x-bt-brainstore-duration-ms: + - '367' + x-bt-cursor: + - anTFU2RhAAA + x-bt-internal-trace-id: + - 6a74c55400000000584f1f0c8024bdc3 + status: + code: 200 + message: OK +- request: + body: '{"query": {"select": [{"op": "star"}], "from": {"op": "function", "name": + {"op": "ident", "name": ["experiment"]}, "args": [{"op": "literal", "value": + "5338e98e-ec96-47bd-8aab-99dc1aefcc74"}]}, "cursor": "anTFU2RhAAA", "limit": + 1000}, "use_columnstore": false, "brainstore_realtime": true, "query_source": + "py_sdk_object_fetcher_experiment"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip + Connection: + - keep-alive + Content-Length: + - '341' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/btql + response: + body: + string: '{"data":[],"schema":{"type":"array","items":{"type":"object","properties":{"_pagination_key":{"description":"A + stable, time-ordered key that can be used to paginate over experiment events. + This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The + transaction id of an event is unique to the network operation that processed + the event insertion. Transaction ids are monotonically increasing over time + and can be used to retrieve a versioned snapshot of the experiment (see the + `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"additionalProperties":false,"properties":{"confidence":{"description":"Optional + confidence score for the classification","type":["number","null"]},"id":{"description":"Stable + classification identifier","type":"string"},"label":{"description":"Original + label of the classification item, which is useful for search and indexing + purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional + metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The + version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The + type of global function. Defaults to ''scorer''.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional + function identifier that produced the classification"}},"required":["id"],"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name + of the file in code where the experiment event was created","type":["string","null"]},"caller_functionname":{"description":"The + function in code which created the experiment event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The + timestamp the experiment event was created","format":"date-time","type":"string"},"error":{"description":"The + error that occurred, if any."},"expected":{"description":"The ground truth + value (an arbitrary, JSON serializable object) that you''d compare to `output` + to determine if your `output` value is correct or not. Braintrust currently + does not compare `output` to `expected` for you, since there are so many different + ways to do that correctly. Instead, these values are just used to help you + navigate your experiments while digging into analyses. However, we may later + use these values to re-score outputs or fine-tune your models"},"experiment_id":{"description":"Unique + identifier for the experiment","format":"uuid","type":"string"},"facets":{"anyOf":[{"additionalProperties":{"type":["string","null"]},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A + unique identifier for the experiment event. If you don''t provide one, Braintrust + will generate one for you","type":"string"},"input":{"description":"The arguments + that uniquely define a test case (an arbitrary, JSON serializable object). + Later on, Braintrust will use the `input` to know whether two test cases are + the same between experiments, so they should not contain experiment-specific + state. A simple rule of thumb is that if you run the same experiment twice, + the `input` should be identical"},"is_root":{"description":"Whether this span + is a root span","type":["boolean","null"]},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The + model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This + metric is deprecated"},"caller_functionname":{"description":"This metric is + deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A + unix timestamp recording when the section of code which produced the experiment + event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A + unix timestamp recording when the section of code which produced the experiment + event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"origin":{"anyOf":[{"description":"Reference + to the original object and event this was copied from.","properties":{"_xact_id":{"description":"Transaction + ID of the original event.","type":["string","null"]},"created":{"description":"Created + timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID + of the original event.","type":"string"},"object_id":{"description":"ID of + the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type + of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"required":["object_type","object_id","id"],"type":"object"},{"type":"null"}]},"output":{"description":"The + output of your application, including post-processing (an arbitrary, JSON + serializable object), that allows you to determine whether the result is correct + or not. For example, in an app that generates SQL queries, the `output` should + be the _result_ of the SQL query generated by the model, not the query itself, + because there may be multiple valid queries that answer a single question"},"project_id":{"description":"Unique + identifier for the project that the experiment belongs under","format":"uuid","type":"string"},"root_span_id":{"description":"A + unique identifier for the trace this experiment event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"additionalProperties":{},"description":"Human-identifying + attributes of the span, such as name, type, etc.","properties":{"name":{"description":"Name + of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A + unique identifier used to link different experiment events together as part + of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) + for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}}},"realtime_state":{"type":"on","minimum_xact_id":"1000197642006809586","read_bytes":8418,"actual_xact_id":"1000197642012355312"},"freshness_state":{"last_processed_xact_id":"1000197642006809586","last_considered_xact_id":"1000197642012355312"},"warnings":[]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Thu, 06 Aug 2026 17:33:09 GMT + Via: + - 1.1 8250156022879efefd7a589c8ba8c706.cloudfront.net (CloudFront), 1.1 16808c837fedc33331e77d172952efee.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - Ydu8C9Wjgyck2pS3Z4TV9-9zm_EPiERSebeum5ZwFQgb4L9fRmQ9aw== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a74c554-6b5c5e3a70e457a73f3dc58b;Parent=439d1ffd7fbabf77;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - private, no-cache + content-length: + - '7788' + vary: + - Origin + x-amz-apigw-id: + - BsPFWEzpoAMEnxA= + x-amzn-RequestId: + - 607fed12-e3e8-4018-8496-243570fe9020 + x-bt-api-duration-ms: + - '147' + x-bt-brainstore-duration-ms: + - '73' + x-bt-internal-trace-id: + - 6a74c5540000000069b34c8d4654c9ea + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/harbor/compat.py b/py/src/braintrust/integrations/harbor/compat.py new file mode 100644 index 00000000..66e6546e --- /dev/null +++ b/py/src/braintrust/integrations/harbor/compat.py @@ -0,0 +1,272 @@ +"""The isolated Harbor-version compatibility boundary.""" + +# Harbor is optional and only supports Python 3.12+, while pylint runs across +# Braintrust's full Python matrix without installing Harbor. +# pylint: disable=import-error + +import json +import logging +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .identity import logical_task_key + + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class TaskData: + logical_key: str + source: str + name: str + input: dict[str, Any] + expected: Any + metadata: dict[str, Any] + digest: str | None + schema_version: str | None + task_dir: Path | None + + +@dataclass(frozen=True) +class TrialPlan: + trial_name: str + trial_config: Any + trial_lock: Any + task: TaskData + attempt_index: int + + +@dataclass(frozen=True) +class JobSnapshot: + job_id: str + job_name: str + job_dir: Path + job_config: Any + job_lock: Any + plans: tuple[TrialPlan, ...] + is_resuming: bool = False + + +def _task_download_path(job: Any, trial_config: Any) -> Path | None: + downloads = getattr(job, "_task_download_results", {}) + try: + result = downloads[trial_config.task.get_task_id()] + except (KeyError, AttributeError): + return None + path = getattr(result, "path", None) + return Path(path) if path is not None else None + + +def _task_data(trial_config: Any, trial_lock: Any, task_dir: Path | None) -> TaskData: + task_obj = None + if task_dir is not None: + try: + from harbor.models.task.task import Task + + # Dataset input is task-authored semantics only. Run-specific extra + # instructions are attached to agent_execution by the converter. + task_obj = Task( + task_dir, + disable_verification=bool(getattr(getattr(trial_config, "verifier", None), "disable", False)), + ) + except Exception: + task_obj = None + + task_lock = getattr(trial_lock, "task", None) + source = ( + getattr(getattr(trial_config, "task", None), "source", None) or getattr(task_lock, "source", None) or "adhoc" + ) + name = ( + getattr(task_obj, "name", None) + or getattr(task_lock, "name", None) + or trial_config.task.get_task_id().get_name() + ) + key = logical_task_key(trial_config, trial_lock) + steps = getattr(getattr(task_obj, "config", None), "steps", None) or [] + if task_obj is not None and steps: + canonical_input = { + "task": name, + "steps": [{"name": step.name, "instruction": task_obj.step_instruction(step.name)} for step in steps], + } + else: + canonical_input = { + "task": name, + "instruction": getattr(task_obj, "instruction", ""), + } + + task_config = getattr(task_obj, "config", None) + user_metadata = dict(getattr(task_config, "metadata", None) or {}) + resources: dict[str, Any] = {} + environment = getattr(task_config, "environment", None) + if environment is not None: + for field in ("cpus", "memory_mb", "storage_mb", "gpus", "tpu", "os"): + value = getattr(environment, field, None) + if value is not None: + resources[field] = getattr(value, "value", value) + metadata = { + "harbor": { + "source": source, + "logical_task_key": key, + "task_digest": getattr(task_lock, "digest", None), + "schema_version": getattr(task_config, "schema_version", None), + "resources": resources, + "custom": user_metadata, + } + } + return TaskData( + logical_key=key, + source=source, + name=name, + input=canonical_input, + expected=None, + metadata=metadata, + digest=getattr(task_lock, "digest", None), + schema_version=getattr(task_config, "schema_version", None), + task_dir=task_dir, + ) + + +def snapshot_job(job: Any) -> JobSnapshot: + """Feature-detect the read-only resolved-plan fields allowed by the design.""" + trial_configs = tuple(getattr(job, "_trial_configs")) + if not trial_configs: + raise ValueError("Harbor job has no resolved trial configurations") + + job_lock = getattr(job, "_job_lock", None) + if job_lock is None: + from harbor.models.job.lock import build_job_lock + + job_lock = build_job_lock( + config=job.config, + trial_configs=trial_configs, + task_download_results=getattr(job, "_task_download_results"), + ) + locks = tuple(job_lock.trials) + if len(locks) != len(trial_configs): + raise ValueError("Harbor trial plan and lock have different lengths") + + attempts: dict[tuple[str, str], int] = {} + plans: list[TrialPlan] = [] + for trial_config, trial_lock in zip(trial_configs, locks): + task_dir = _task_download_path(job, trial_config) + task = _task_data(trial_config, trial_lock, task_dir) + agent = trial_config.agent + attempt_key = (task.logical_key, json.dumps(agent.model_dump(mode="json", exclude_none=True), sort_keys=True)) + attempt_index = attempts.get(attempt_key, 0) + attempts[attempt_key] = attempt_index + 1 + plans.append(TrialPlan(trial_config.trial_name, trial_config, trial_lock, task, attempt_index)) + + return JobSnapshot( + job_id=str(job.id), + job_name=str(job.config.job_name), + job_dir=Path(job.job_dir), + job_config=job.config, + job_lock=job_lock, + plans=tuple(plans), + is_resuming=bool(getattr(job, "is_resuming", False)), + ) + + +def trial_directory(result: Any) -> Path: + config = result.config + return Path(config.trials_dir) / result.trial_name + + +def _step_paths(result: Any, *parts: str) -> list[tuple[str | None, Path]]: + """Resolve one per-step file, encoding Harbor's on-disk step layout once. + + The step name travels with the path so callers can attribute a file to the step + that produced it; it is None for a single-phase trial, which has no steps/ level. + """ + base = trial_directory(result) + if result.step_results: + return [(step.step_name, base.joinpath("steps", step.step_name, *parts)) for step in result.step_results] + return [(None, base.joinpath(*parts))] + + +def trajectory_paths(result: Any) -> list[tuple[str | None, Path]]: + return _step_paths(result, "agent", "trajectory.json") + + +def reward_details_paths(result: Any) -> list[tuple[str | None, Path]]: + return _step_paths(result, "verifier", "reward-details.json") + + +def artifact_manifest_paths(result: Any) -> list[tuple[str | None, Path]]: + return _step_paths(result, "artifacts", "manifest.json") + + +def load_backfill_snapshot(job_dir: str | Path) -> tuple[JobSnapshot, Any]: + """Load persisted Harbor models for offline backfill.""" + from harbor.models.job.config import JobConfig + from harbor.models.job.lock import JobLock, TrialLock + from harbor.models.job.result import JobResult + from harbor.models.trial.result import TrialResult + + directory = Path(job_dir).expanduser().resolve() + config = JobConfig.model_validate_json((directory / "config.json").read_text()) + lock = JobLock.model_validate_json((directory / "lock.json").read_text()) + job_result = JobResult.model_validate_json((directory / "result.json").read_text()) + results: list[Any] = [] + result_paths = { + result_path.parent: result_path + for pattern in ("*/results.json", "*/result.json") + for result_path in sorted(directory.glob(pattern)) + } + for result_path in result_paths.values(): + try: + trial_result = TrialResult.model_validate_json(result_path.read_text()) + # A job directory may be moved before backfill. Resolve trial files + # relative to the directory being backfilled, not the old jobs_dir. + trial_result.config.trials_dir = directory + results.append(trial_result) + except Exception: + logger.warning("Skipping unreadable Harbor trial result %s", result_path, exc_info=True) + continue + if not results and job_result.trial_results: + results = list(job_result.trial_results) + job_result.trial_results = results + + # TrialResult.task_checksum is a task-directory hash while TaskLock.digest is + # a content digest, so the two are not comparable. Correlate on task name and + # accept no lock rather than attributing an unrelated trial's lock, whose task + # identity and skills would silently collapse distinct tasks and partitions. + lock_by_name: dict[str, list[Any]] = {} + for item in lock.trials: + lock_by_name.setdefault(item.task.name, []).append(item) + attempts: dict[tuple[str, str], int] = {} + plans: list[TrialPlan] = [] + for result in results: + trial_lock_path = trial_directory(result) / "lock.json" + if trial_lock_path.exists(): + trial_lock = TrialLock.model_validate_json(trial_lock_path.read_text()) + else: + candidates = lock_by_name.get(result.task_name) or [] + trial_lock = candidates.pop(0) if candidates else None + if trial_lock is None: + logger.warning("No job lock entry matches Harbor task %r; continuing without it", result.task_name) + task_dir = None + try: + candidate = result.config.task.get_task_id().get_local_path() + task_dir = candidate if candidate.exists() else None + except Exception: + pass + task = _task_data(result.config, trial_lock, task_dir) + agent_key = json.dumps(result.config.agent.model_dump(mode="json", exclude_none=True), sort_keys=True) + attempt_key = (task.logical_key, agent_key) + attempt_index = attempts.get(attempt_key, 0) + attempts[attempt_key] = attempt_index + 1 + plans.append(TrialPlan(result.trial_name, result.config, trial_lock, task, attempt_index)) + + snapshot = JobSnapshot( + job_id=str(job_result.id), + job_name=config.job_name, + job_dir=directory, + job_config=config, + job_lock=lock, + plans=tuple(plans), + is_resuming=True, + ) + return snapshot, job_result diff --git a/py/src/braintrust/integrations/harbor/config.py b/py/src/braintrust/integrations/harbor/config.py new file mode 100644 index 00000000..97254444 --- /dev/null +++ b/py/src/braintrust/integrations/harbor/config.py @@ -0,0 +1,204 @@ +"""Configuration for the Harbor job plugin.""" + +import fnmatch +import json +import math +import os +from dataclasses import dataclass, field, fields +from typing import Any + + +_UNSET = object() +_PREFIX = "HARBOR_BRAINTRUST_" + + +def _environment_value(name: str, default: Any) -> Any: + names = [f"{_PREFIX}{name.upper()}"] + if name == "project_name": + names.append(f"{_PREFIX}PROJECT") + for environment_name in names: + value = os.environ.get(environment_name) + if value is not None: + return value + return default + + +def _resolve(value: Any, name: str, default: Any) -> Any: + return _environment_value(name, default) if value is _UNSET else value + + +def _parse_bool(value: Any, name: str) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + raise ValueError(f"{name} must be a boolean") + + +def _parse_int(value: Any, name: str) -> int: + if isinstance(value, bool): + raise ValueError(f"{name} must be a non-negative integer") + try: + parsed = int(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be a non-negative integer") from exc + if parsed < 0: + raise ValueError(f"{name} must be a non-negative integer") + return parsed + + +def _parse_json(value: Any, name: str, expected_type: type) -> Any: + if value is None: + return None + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError as exc: + raise ValueError(f"{name} must be valid JSON") from exc + if not isinstance(value, expected_type): + raise ValueError(f"{name} must be a JSON {expected_type.__name__}") + return value + + +def _parse_patterns(value: Any, name: str) -> tuple[str, ...]: + if value is None: + return () + value = _parse_json(value, name, list) if isinstance(value, str) else value + if not isinstance(value, (list, tuple)) or not all(isinstance(item, str) and item for item in value): + raise ValueError(f"{name} must be a JSON array of non-empty strings") + return tuple(value) + + +def _patterns_overlap(left: str, right: str) -> bool: + # Exact equality and either pattern matching the other catch all useful, + # deterministic overlap cases without pretending to solve glob intersection. + return left == right or fnmatch.fnmatchcase(left, right) or fnmatch.fnmatchcase(right, left) + + +@dataclass(frozen=True) +class PluginConfig: + project_name: str | None = None + project_id: str | None = None + experiment_prefix: str | None = None + base_experiment_name: str | None = None + base_experiment_id: str | None = None + dataset_mode: str = "sync" + dataset_name: str | None = None + trajectory_mode: str = "atif" + content_mode: str = "messages" + include_custom_metadata: bool = True + max_custom_metadata_bytes: int = 100_000 + score_keys: tuple[str, ...] = () + metric_keys: tuple[str, ...] = () + reward_rules: dict[str, dict[str, Any]] = field(default_factory=dict) + classifier_rules: dict[str, str] = field(default_factory=dict) + invalid_score_policy: str = "metric" + include_tracebacks: bool = False + attachments: str = "verifier-details" + artifact_include: tuple[str, ...] = () + max_attachment_bytes: int = 5_000_000 + max_total_attachment_bytes: int = 20_000_000 + max_content_bytes: int = 20_000 + log_retry_attempts: bool = False + strict: bool = False + redact_patterns: tuple[str, ...] = () + + @classmethod + def from_options(cls, **options: Any) -> "PluginConfig": + defaults = cls() + values: dict[str, Any] = {} + for config_field in fields(defaults): + name = config_field.name + values[name] = _resolve(options.get(name, _UNSET), name, getattr(defaults, name)) + + for name in ( + "include_custom_metadata", + "include_tracebacks", + "log_retry_attempts", + "strict", + ): + values[name] = _parse_bool(values[name], name) + for name in ( + "max_custom_metadata_bytes", + "max_attachment_bytes", + "max_total_attachment_bytes", + "max_content_bytes", + ): + values[name] = _parse_int(values[name], name) + for name in ("score_keys", "metric_keys", "artifact_include", "redact_patterns"): + values[name] = _parse_patterns(values[name], name) + for name in ("reward_rules", "classifier_rules"): + values[name] = _parse_json(values[name], name, dict) or {} + + config = cls(**values) + config.validate() + return config + + def validate(self) -> None: + if self.project_name and self.project_id: + raise ValueError("project_name and project_id are mutually exclusive") + if self.base_experiment_name and self.base_experiment_id: + raise ValueError("base_experiment_name and base_experiment_id are mutually exclusive") + if self.dataset_mode not in {"sync", "none"}: + raise ValueError("dataset_mode must be 'sync' or 'none'; 'existing' is not supported yet") + if self.dataset_name and self.dataset_mode != "sync": + raise ValueError("dataset_name requires dataset_mode='sync'") + if self.trajectory_mode not in {"atif", "summary", "native"}: + raise ValueError("trajectory_mode must be 'atif', 'summary', or 'native'") + # 'full' is 'messages' plus fields the instrumentation contract explicitly + # allows. No such field exists yet, so the two capture the same payload. + if self.content_mode not in {"metadata", "messages", "full"}: + raise ValueError("content_mode must be 'metadata', 'messages', or 'full'") + if self.log_retry_attempts: + raise ValueError("log_retry_attempts=True is not implemented; only the final attempt is logged") + if self.invalid_score_policy not in {"metric", "drop", "error"}: + raise ValueError("invalid_score_policy must be 'metric', 'drop', or 'error'") + if self.attachments not in {"none", "verifier-details", "all"}: + raise ValueError("attachments must be 'none', 'verifier-details', or 'all'") + if self.artifact_include and self.attachments != "all": + raise ValueError("artifact_include requires attachments='all'") + if self.max_total_attachment_bytes < self.max_attachment_bytes: + raise ValueError("max_total_attachment_bytes must be at least max_attachment_bytes") + + for score_pattern in self.score_keys: + for metric_pattern in self.metric_keys: + if _patterns_overlap(score_pattern, metric_pattern): + raise ValueError(f"score_keys and metric_keys overlap: {score_pattern!r}, {metric_pattern!r}") + + for key, rule in self.reward_rules.items(): + if not isinstance(key, str) or not key or not isinstance(rule, dict): + raise ValueError("reward_rules must map non-empty strings to objects") + rule_type = rule.get("type") + if rule_type not in {"score", "metric"}: + raise ValueError(f"reward_rules[{key!r}].type must be 'score' or 'metric'") + if rule_type == "metric" and any(field in rule for field in ("direction", "min", "max", "score_name")): + raise ValueError(f"metric reward rule {key!r} cannot define score normalization") + if rule_type == "score": + direction = rule.get("direction", "maximize") + if direction not in {"maximize", "minimize"}: + raise ValueError(f"reward_rules[{key!r}].direction must be 'maximize' or 'minimize'") + has_min, has_max = "min" in rule, "max" in rule + if has_min != has_max: + raise ValueError(f"reward_rules[{key!r}] must define both min and max") + if has_min: + minimum, maximum = rule["min"], rule["max"] + if ( + isinstance(minimum, bool) + or isinstance(maximum, bool) + or not isinstance(minimum, (int, float)) + or not isinstance(maximum, (int, float)) + or not math.isfinite(float(minimum)) + or not math.isfinite(float(maximum)) + or minimum >= maximum + ): + raise ValueError(f"reward_rules[{key!r}] requires finite min < max") + + if not all( + isinstance(name, str) and name and isinstance(path, str) and path + for name, path in self.classifier_rules.items() + ): + raise ValueError("classifier_rules must map non-empty names to non-empty JSON paths") diff --git a/py/src/braintrust/integrations/harbor/identity.py b/py/src/braintrust/integrations/harbor/identity.py new file mode 100644 index 00000000..5eabe6b1 --- /dev/null +++ b/py/src/braintrust/integrations/harbor/identity.py @@ -0,0 +1,305 @@ +"""Deterministic identity and privacy-safe normalization helpers.""" + +import hashlib +import json +import re +from dataclasses import dataclass +from pathlib import Path, PurePath +from typing import Any +from urllib.parse import urlsplit, urlunsplit +from uuid import UUID, uuid5 + + +PLUGIN_NAMESPACE = UUID("67ea9f8a-e42a-5f31-96d8-85bcf27ca4c9") +_SECRET_KEY = re.compile(r"(?:api[_-]?key|token|secret|password|credential|authorization|cookie)", re.IGNORECASE) +_ABSOLUTE_WINDOWS_PATH = re.compile(r"^[a-zA-Z]:[\\/]") +_TEMPLATE = re.compile(r"^\$\{[A-Za-z_][A-Za-z0-9_]*(?::-[^}]*)?\}$") +_KEY_SEGMENT = re.compile(r"[^0-9A-Za-z]+|(?<=[a-z0-9])(?=[A-Z])") +# Keys built from a counting or budgeting word are measurements, not credentials. +# "max_tokens" and "total_tokens" match the secret pattern as substrings, and +# redacting them both destroys usage metadata and collapses agent configurations +# that differ only by a token budget into one partition. +_COUNTER_SEGMENTS = frozenset( + { + "average", + "avg", + "budget", + "cache", + "cached", + "completion", + "count", + "counts", + "input", + "limit", + "max", + "maximum", + "min", + "minimum", + "num", + "output", + "per", + "prompt", + "reasoning", + "remaining", + "size", + "sum", + "total", + "usage", + "used", + "window", + } +) + + +@dataclass(frozen=True) +class NormalizedValue: + value: Any + warnings: tuple[str, ...] = () + + @property + def complete(self) -> bool: + """Report whether the value survived normalization intact. + + Every warning this module emits names data it dropped, truncated, or + replaced, so the absence of warnings is the completeness signal. + """ + return not self.warnings + + +def canonical_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False) + + +def stable_hash(value: Any) -> str: + return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() + + +def deterministic_id(scope: str, value: str) -> str: + return str(uuid5(PLUGIN_NAMESPACE, f"{scope}:{value}")) + + +def dataset_record_id(dataset_scope: str, logical_task_key: str) -> str: + return deterministic_id("dataset-record", f"{dataset_scope}:{logical_task_key}") + + +def child_span_id(trial_id: str, semantic_path: str) -> str: + try: + namespace = UUID(str(trial_id)) + except ValueError: + namespace = uuid5(PLUGIN_NAMESPACE, str(trial_id)) + return str(uuid5(namespace, semantic_path)) + + +def _is_absolute_path(value: str) -> bool: + return value.startswith(("/", "~/", "file://")) or bool(_ABSOLUTE_WINDOWS_PATH.match(value)) + + +def _key_segments(key: str) -> set[str]: + return {segment.lower() for segment in _KEY_SEGMENT.split(key) if segment} + + +def _is_secret_key(key: str, value: Any) -> bool: + """Report whether a key names a credential whose value must not be logged.""" + if value is None or isinstance(value, (bool, int, float)): + # A number is never a credential, so redacting it can only lose data. + return False + if not _SECRET_KEY.search(key): + return False + return _key_segments(key).isdisjoint(_COUNTER_SEGMENTS) + + +def _json_size(value: Any) -> int: + try: + return len(canonical_json(value).encode("utf-8")) + except (TypeError, ValueError): + return 10**18 + + +def normalize_json( + value: Any, + *, + max_bytes: int, + redact_patterns: tuple[str, ...] = (), + max_depth: int = 8, + redact_absolute_paths: bool = True, +) -> NormalizedValue: + """Normalize untrusted metadata while preserving JSON types and nulls. + + Set ``redact_absolute_paths=False`` for payloads produced inside the task + sandbox: their absolute paths are container paths the agent actually operated + on, so redacting them erases the substance of filesystem tool calls. + """ + warnings: list[str] = [] + compiled_patterns = tuple(re.compile(pattern) for pattern in redact_patterns) + + def walk(item: Any, path: str, depth: int, key: str | None = None) -> Any: + if depth > max_depth: + warnings.append(f"dropped {path}: depth limit") + return "[DROPPED: depth limit]" + if key is not None and _is_secret_key(key, item): + if isinstance(item, str) and _TEMPLATE.match(item): + return item + warnings.append(f"redacted {path}: sensitive key") + return "[REDACTED]" + if item is None or isinstance(item, (bool, int, float)): + return item + if isinstance(item, str): + if redact_absolute_paths and _is_absolute_path(item): + warnings.append(f"dropped {path}: absolute path") + return "[REDACTED PATH]" + result = item + for pattern in compiled_patterns: + result = pattern.sub("[REDACTED]", result) + return result + if isinstance(item, PurePath): + raw = str(item) + if redact_absolute_paths and (item.is_absolute() or _is_absolute_path(raw)): + warnings.append(f"dropped {path}: absolute path") + return "[REDACTED PATH]" + return item.as_posix() + if isinstance(item, dict): + normalized: dict[str, Any] = {} + for raw_key, child in item.items(): + child_key = str(raw_key) + child_path = f"{path}.{child_key}" if path else child_key + normalized[child_key] = walk(child, child_path, depth + 1, child_key) + return normalized + if isinstance(item, (list, tuple)): + return [walk(child, f"{path}[{index}]", depth + 1) for index, child in enumerate(item)] + model_dump = getattr(item, "model_dump", None) + if callable(model_dump): + try: + return walk(model_dump(mode="json", exclude_none=False), path, depth) + except Exception: + pass + warnings.append(f"dropped {path}: unsupported type {type(item).__name__}") + return f"[DROPPED: {type(item).__name__}]" + + normalized = walk(value, "", 0) + if _json_size(normalized) <= max_bytes: + return NormalizedValue(normalized, tuple(warnings)) + + # Fitting a container to a byte budget needs each entry's serialized size, not + # a fresh serialization of every prefix: the payload is serialized again on its + # way to Braintrust, so measuring it here more than once is pure overhead. + # Canonical JSON adds two braces or brackets plus one separator between + # entries, and one colon per object key, so entry sizes accumulate exactly. + warnings.append(f"truncated value: exceeded {max_bytes} bytes") + if isinstance(normalized, dict): + bounded: dict[str, Any] = {} + total = 2 + for key in sorted(normalized): + entry = _json_size(key) + 1 + _json_size(normalized[key]) + (1 if bounded else 0) + if total + entry > max_bytes: + warnings.append(f"dropped {key}: size limit") + continue + total += entry + bounded[key] = normalized[key] + normalized = bounded + elif isinstance(normalized, str): + normalized = normalized.encode("utf-8")[:max_bytes].decode("utf-8", errors="ignore") + elif isinstance(normalized, list): + # Keep the leading elements so an oversized list stays a list. Replacing + # the whole value with a placeholder string would change its JSON type. + total = 2 + kept = 0 + for index, item in enumerate(normalized): + total += _json_size(item) + (1 if index else 0) + if total > max_bytes: + warnings.append(f"dropped [{index}:]: size limit") + break + kept = index + 1 + normalized = normalized[:kept] + else: + normalized = "[DROPPED: size limit]" + return NormalizedValue(normalized, tuple(warnings)) + + +def safe_git_url(value: str) -> str: + parsed = urlsplit(value) + hostname = parsed.hostname or "" + if parsed.port: + hostname = f"{hostname}:{parsed.port}" + return urlunsplit((parsed.scheme, hostname, parsed.path.rstrip("/"), "", "")) + + +def logical_task_key(task_config: Any, task_lock: Any | None = None) -> str: + """Choose a stable logical task identity without exposing local paths.""" + task = getattr(task_config, "task", None) + if task is not None: + name = getattr(task, "name", None) + ref = getattr(task, "ref", None) + if name: + return f"package:{name}@{ref or 'default'}" + + git_url = getattr(task, "git_url", None) + path = getattr(task, "path", None) + if git_url: + relative = Path(path).as_posix().lstrip("/") if path is not None else "" + return f"git:{safe_git_url(git_url)}#{relative}" + + lock_name = getattr(getattr(task_lock, "task", task_lock), "name", None) + source = getattr(task, "source", None) or getattr(getattr(task_lock, "task", task_lock), "source", None) + if lock_name: + return f"harbor:{source or 'adhoc'}:{lock_name}" + name = Path(path).name if path is not None else "task" + return f"local:{source or 'adhoc'}:{name}" + + +def dataset_scope(source: str) -> str: + """Scope a dataset by task source alone. + + The resolved task subset is deliberately excluded. It changes whenever a job + runs a subset of the source's tasks, or whenever backfill cannot read one + trial's result, and because this scope also feeds record IDs, partition keys, + and the experiment name, including it would fork a new dataset and experiment + instead of reconciling the existing ones. Records are keyed per logical task, + so a narrower run upserts a subset of rows. + """ + return f"{source}:tasks" + + +def dataset_display_name(source: str, prefix: str = "harbor") -> str: + return f"{prefix} · {source}" + + +def semantic_agent_config(agent: Any, skills: list[Any]) -> dict[str, Any]: + def safe_env(raw: Any) -> dict[str, str]: + result: dict[str, str] = {} + for key, value in (raw or {}).items(): + value = str(value) + if _is_secret_key(str(key), value): + result[str(key)] = value if _TEMPLATE.match(value) else f"${{{key}}}" + else: + result[str(key)] = value + return result + + raw = { + "name": getattr(agent, "name", None), + "import_path": getattr(agent, "import_path", None), + "model": getattr(agent, "model_name", None), + "kwargs": getattr(agent, "kwargs", None) or {}, + "env": safe_env(getattr(agent, "env", None)), + "mcp_servers": getattr(agent, "mcp_servers", None) or [], + "resume_trajectory": bool(getattr(agent, "resume_trajectory", False)), + "load_trajectory": getattr(agent, "load_trajectory", None), + "skills": sorted( + [ + { + "name": getattr(skill, "name", None), + "digest": getattr(skill, "digest", None), + "git_url": safe_git_url(str(getattr(skill, "git_url"))) + if getattr(skill, "git_url", None) + else None, + "git_commit_id": getattr(skill, "git_commit_id", None), + } + for skill in skills + ], + key=canonical_json, + ), + } + return normalize_json(raw, max_bytes=200_000).value + + +def partition_key(dataset_key: str, agent_config: dict[str, Any]) -> str: + return stable_hash({"dataset": dataset_key, "agent": agent_config}) diff --git a/py/src/braintrust/integrations/harbor/plugin.py b/py/src/braintrust/integrations/harbor/plugin.py new file mode 100644 index 00000000..83e794af --- /dev/null +++ b/py/src/braintrust/integrations/harbor/plugin.py @@ -0,0 +1,968 @@ +"""Native Braintrust job plugin for Harbor.""" + +# Harbor is optional and only supports Python 3.12+, while pylint runs across +# Braintrust's full Python matrix without installing Harbor. +# pylint: disable=import-error + +import asyncio +import fnmatch +import json +import logging +import os +from dataclasses import dataclass, field, fields +from datetime import datetime +from pathlib import Path +from typing import Any + +from braintrust.logger import Attachment, flush, init, init_dataset +from exceptiongroup import ExceptionGroup + +from .atif import _INSTRUMENTATION, ATIFImportResult, import_trajectory, summarize_trajectory +from .compat import ( + JobSnapshot, + TrialPlan, + artifact_manifest_paths, + load_backfill_snapshot, + reward_details_paths, + snapshot_job, + trajectory_paths, +) +from .config import _UNSET, PluginConfig +from .identity import ( + canonical_json, + child_span_id, + dataset_display_name, + dataset_record_id, + dataset_scope, + normalize_json, + partition_key, + semantic_agent_config, +) +from .rewards import classify_rewards, extract_json_path, validate_classifications +from .state import ( + JobEvent, + JobMachine, + TrialEvent, + TrialEventKind, + TrialMachine, + TrialStatus, + accepts_trial_events, + can_reconcile, + reduce_job, + reduce_trial, +) + + +logger = logging.getLogger(__name__) +_PLUGIN_VERSION = "1" +_MANIFEST_VERSION = 1 + + +@dataclass +class DatasetBinding: + scope: str + dataset: Any = None + origins: dict[str, dict[str, Any]] = field(default_factory=dict) + error: str | None = None + + +@dataclass +class Partition: + key: str + name: str + dataset_scope: str + experiment: Any = None + experiment_id: str | None = None + + +@dataclass +class RuntimeState: + snapshot: JobSnapshot + plan_by_trial: dict[str, TrialPlan] + partition_by_trial: dict[str, Partition] + datasets: dict[str, DatasetBinding] + partitions: dict[str, Partition] + + +def _seconds(value: Any, fallback: float) -> float: + if isinstance(value, datetime): + # Harbor trial timestamps are timezone-aware, but job timestamps are + # currently naive local datetimes. datetime.timestamp() preserves both + # conventions; assigning UTC to a naive value shifts non-UTC jobs. + return value.timestamp() + return fallback + + +def _timing(value: Any, default_start: float, default_end: float) -> tuple[float, float]: + start = _seconds(getattr(value, "started_at", None), default_start) + end = _seconds(getattr(value, "finished_at", None), default_end) + if end < start: + end = start + return start, end + + +def _exception(result: Any, include_traceback: bool) -> tuple[str | None, str | None]: + info = getattr(result, "exception_info", None) + if info is None: + return None, None + error = f"{info.exception_type}: {info.exception_message}" + traceback_value = info.exception_traceback if include_traceback else None + return error, traceback_value + + +def _answer_from_metadata(result: Any) -> Any: + contexts = [] + if getattr(result, "agent_result", None) is not None: + contexts.append(result.agent_result) + for step in getattr(result, "step_results", None) or []: + if getattr(step, "agent_result", None) is not None: + contexts.append(step.agent_result) + for context in reversed(contexts): + metadata = getattr(context, "metadata", None) + if not isinstance(metadata, dict): + continue + for key in ("standardized_answer", "final_answer", "answer", "output", "response"): + if key in metadata: + return metadata[key] + return None + + +def _rewards(result: Any) -> dict[str, Any]: + verifier = getattr(result, "verifier_result", None) + raw = getattr(verifier, "rewards", None) + return dict(raw or {}) + + +def _by_step(items: list[tuple[str | None, Any]], default_key: str, *, keep_single_name: bool = True) -> Any: + """Collapse per-step values into one metadata value, or None when there are none. + + ``keep_single_name`` decides what a single value from a *named* step becomes. + Trajectory metadata keeps the label, because which step produced the totals is + part of the answer; the eval-root output drops it, because a single-step trial's + answer should read as the answer rather than as a one-entry map. + """ + if not items: + return None + if len(items) == 1 and (items[0][0] is None or not keep_single_name): + return items[0][1] + return {name or default_key: value for name, value in items} + + +def _step_label(step_name: str | None, path: Path) -> str: + return path.name if step_name is None else f"{step_name}/{path.name}" + + +def _read_json_summary(entries: list[tuple[str | None, Path]], max_bytes: int) -> tuple[Any, list[str]]: + summaries: list[tuple[str | None, Any]] = [] + warnings: list[str] = [] + for step_name, path in entries: + label = _step_label(step_name, path) + try: + size = path.stat().st_size + with path.open("rb") as file_obj: + data = file_obj.read(min(size, max_bytes) + 1) + if len(data) > max_bytes: + warnings.append(f"{label} omitted: size limit") + continue + summaries.append((step_name, json.loads(data))) + except FileNotFoundError: + continue + except (OSError, json.JSONDecodeError) as exc: + warnings.append(f"could not read {label}: {exc}") + return _by_step(summaries, "manifest"), warnings + + +def _artifact_attachments(result: Any, config: PluginConfig) -> tuple[dict[str, Attachment], list[str]]: + if config.attachments != "all" or not config.artifact_include: + return {}, [] + attachments: dict[str, Attachment] = {} + warnings: list[str] = [] + total = 0 + for step_name, manifest_path in artifact_manifest_paths(result): + root = manifest_path.parent.resolve() + if not root.exists(): + continue + for path in sorted(root.rglob("*")): + if not path.is_file() or path.name == "manifest.json" or path.is_symlink(): + continue + try: + resolved = path.resolve() + relative = resolved.relative_to(root).as_posix() + except (OSError, ValueError): + warnings.append(f"artifact {path.name} omitted: unsafe path") + continue + if not any(fnmatch.fnmatchcase(relative, pattern) for pattern in config.artifact_include): + continue + # Each step has its own artifacts root, so the relative path alone + # collides whenever two steps collect the same file name. + key = relative if step_name is None else f"{step_name}/{relative}" + try: + size = resolved.stat().st_size + if size > config.max_attachment_bytes or total + size > config.max_total_attachment_bytes: + warnings.append(f"artifact {key} omitted: attachment size limit") + continue + data = resolved.read_bytes() + except OSError as exc: + warnings.append(f"artifact {key} omitted: {exc}") + continue + total += len(data) + attachments[key] = Attachment( + data=data, + filename=resolved.name, + content_type="application/octet-stream", + ) + return attachments, warnings + + +def _attachment( + entries: list[tuple[str | None, Path]], config: PluginConfig +) -> tuple[Attachment | None, Any, list[str]]: + if config.attachments == "none": + return None, None, [] + total = 0 + complete: list[tuple[str | None, Any]] = [] + warnings: list[str] = [] + filename = "details.json" + for step_name, path in entries: + label = _step_label(step_name, path) + filename = path.name + try: + data = path.read_bytes() + except FileNotFoundError: + continue + except OSError as exc: + warnings.append(f"could not read {label}: {exc}") + continue + if len(data) > config.max_attachment_bytes or total + len(data) > config.max_total_attachment_bytes: + warnings.append(f"{label} omitted: attachment size limit") + continue + try: + parsed = json.loads(data) + except json.JSONDecodeError: + warnings.append(f"{label} is not valid JSON") + continue + normalized = normalize_json( + parsed, + max_bytes=config.max_attachment_bytes, + redact_patterns=config.redact_patterns, + max_depth=20, + ) + warnings.extend(normalized.warnings) + complete.append((step_name, normalized.value)) + total += len(data) + summary = _by_step(complete, "details") + if summary is None: + return None, None, warnings + attachment_data = (canonical_json(summary) + "\n").encode() + # One serialized payload is bounded by the per-file limit, not the job total. + if len(attachment_data) > config.max_attachment_bytes: + warnings.append(f"{filename} omitted after redaction: attachment size limit") + return None, summary, warnings + return ( + Attachment(data=attachment_data, filename=filename, content_type="application/json"), + summary, + warnings, + ) + + +class HarborPlugin: + """Harbor plugin that reconciles final trials into Braintrust experiments.""" + + def __init__( + self, + project_name: Any = _UNSET, + project_id: Any = _UNSET, + experiment_prefix: Any = _UNSET, + base_experiment_name: Any = _UNSET, + base_experiment_id: Any = _UNSET, + dataset_mode: Any = _UNSET, + dataset_name: Any = _UNSET, + trajectory_mode: Any = _UNSET, + content_mode: Any = _UNSET, + include_custom_metadata: Any = _UNSET, + max_custom_metadata_bytes: Any = _UNSET, + score_keys: Any = _UNSET, + metric_keys: Any = _UNSET, + reward_rules: Any = _UNSET, + classifier_rules: Any = _UNSET, + invalid_score_policy: Any = _UNSET, + include_tracebacks: Any = _UNSET, + attachments: Any = _UNSET, + artifact_include: Any = _UNSET, + max_attachment_bytes: Any = _UNSET, + max_total_attachment_bytes: Any = _UNSET, + max_content_bytes: Any = _UNSET, + log_retry_attempts: Any = _UNSET, + strict: Any = _UNSET, + **kwargs: Any, + ) -> None: + options = { + "project_name": project_name, + "project_id": project_id, + "experiment_prefix": experiment_prefix, + "base_experiment_name": base_experiment_name, + "base_experiment_id": base_experiment_id, + "dataset_mode": dataset_mode, + "dataset_name": dataset_name, + "trajectory_mode": trajectory_mode, + "content_mode": content_mode, + "include_custom_metadata": include_custom_metadata, + "max_custom_metadata_bytes": max_custom_metadata_bytes, + "score_keys": score_keys, + "metric_keys": metric_keys, + "reward_rules": reward_rules, + "classifier_rules": classifier_rules, + "invalid_score_policy": invalid_score_policy, + "include_tracebacks": include_tracebacks, + "attachments": attachments, + "artifact_include": artifact_include, + "max_attachment_bytes": max_attachment_bytes, + "max_total_attachment_bytes": max_total_attachment_bytes, + "max_content_bytes": max_content_bytes, + "log_retry_attempts": log_retry_attempts, + "strict": strict, + **kwargs, + } + unknown = set(options) - {config_field.name for config_field in fields(PluginConfig)} + if unknown: + raise TypeError(f"Unexpected HarborPlugin options: {', '.join(sorted(unknown))}") + self.config = PluginConfig.from_options(**options) + self._job_machine = JobMachine() + self._trial_machines: dict[str, TrialMachine] = {} + self._trial_locks: dict[str, asyncio.Lock] = {} + self._runtime: RuntimeState | None = None + self._snapshot: JobSnapshot | None = None + self._errors: list[str] = [] + self._warnings: list[str] = [] + self._manifest: dict[str, Any] = {} + self._disabled_reason: str | None = None + + async def on_job_start(self, job: Any) -> None: + self._job_machine = reduce_job(self._job_machine, JobEvent.INITIALIZE, strict=self.config.strict) + try: + snapshot = await asyncio.to_thread(snapshot_job, job) + self._snapshot = snapshot + self._runtime = await asyncio.to_thread(self._initialize, snapshot) + for plan in snapshot.plans: + self._trial_machines[plan.trial_name] = TrialMachine(plan.trial_name) + self._trial_locks[plan.trial_name] = asyncio.Lock() + self._register_hooks(job) + self._job_machine = reduce_job(self._job_machine, JobEvent.READY, strict=self.config.strict) + await asyncio.to_thread(self._persist_manifest, False) + except Exception as exc: + self._disable(f"Braintrust initialization failed: {exc}") + if self._snapshot is not None: + try: + await asyncio.to_thread(self._persist_disabled_manifest) + except OSError as persist_exc: + self._errors.append(f"could not persist disabled manifest: {persist_exc}") + if self.config.strict: + raise + + async def on_job_end(self, job_result: Any) -> None: + if self._runtime is None: + return + if not can_reconcile(self._job_machine): + # Initialization failed after the runtime was built. Reconciling now + # would write a full experiment while the manifest reports the sync as + # disabled, and RECONCILE out of a terminal status is not legal. + logger.warning( + "Skipping Braintrust reconciliation while %s: %s", + self._job_machine.status.value, + self._disabled_reason or "job is not active", + ) + return + self._job_machine = reduce_job(self._job_machine, JobEvent.RECONCILE, strict=self.config.strict) + final_names = {result.trial_name for result in job_result.trial_results} + failures: list[BaseException] = [] + + async def reconcile(result: Any) -> None: + try: + await self._dispatch(result.trial_name, TrialEvent(TrialEventKind.FINAL_RESULT, payload=result)) + await asyncio.to_thread(self._sync_final_result, result) + await self._dispatch(result.trial_name, TrialEvent(TrialEventKind.SYNCED)) + except Exception as exc: + failures.append(exc) + self._errors.append(f"trial {result.trial_name}: {exc}") + try: + await self._dispatch(result.trial_name, TrialEvent(TrialEventKind.SYNC_FAILED, payload=str(exc))) + except Exception: + pass + + await asyncio.gather(*(reconcile(result) for result in job_result.trial_results)) + for name in set(self._trial_machines) - final_names: + await self._dispatch(name, TrialEvent(TrialEventKind.OMIT)) + try: + await asyncio.to_thread(flush) + except Exception as exc: + failures.append(exc) + self._errors.append(f"final flush: {exc}") + self._job_machine = reduce_job(self._job_machine, JobEvent.CLOSE, strict=self.config.strict) + try: + await asyncio.to_thread(self._persist_manifest, not failures) + except Exception as exc: + failures.append(exc) + self._errors.append(f"manifest persistence: {exc}") + logger.warning("Could not persist Harbor Braintrust sync manifest", exc_info=True) + if failures and self.config.strict: + # Harbor isolates finalizers, so raising here cannot fail the run; log + # at error level so a strict sync failure is not invisible. Direct + # callers such as backfill still observe the exception. + logger.error("Braintrust Harbor synchronization failed: %s", "; ".join(self._errors)) + raise ExceptionGroup("Braintrust Harbor synchronization failed", failures) + + def _disable(self, message: str) -> None: + self._disabled_reason = message + self._errors.append(message) + self._job_machine = reduce_job(self._job_machine, JobEvent.DISABLE) + logger.warning(message, exc_info=True) + + def _register_hooks(self, job: Any) -> None: + from harbor.trial.hooks import TrialEvent as HarborTrialEvent + + mapping = { + HarborTrialEvent.START: TrialEventKind.START, + HarborTrialEvent.ENVIRONMENT_START: TrialEventKind.ENVIRONMENT_START, + HarborTrialEvent.AGENT_START: TrialEventKind.AGENT_START, + HarborTrialEvent.AGENT_END: TrialEventKind.AGENT_END, + HarborTrialEvent.VERIFICATION_START: TrialEventKind.VERIFICATION_START, + HarborTrialEvent.END: TrialEventKind.END, + HarborTrialEvent.CANCEL: TrialEventKind.CANCEL, + } + max_retries = int(getattr(getattr(job.config, "retry", None), "max_retries", 0) or 0) + for harbor_event, internal_kind in mapping.items(): + + async def callback(event: Any, kind: TrialEventKind = internal_kind) -> None: + try: + machine = self._trial_machines.get(event.trial_name) + retry_predicted = bool( + kind == TrialEventKind.END + and machine is not None + and machine.retry_index < max_retries + and getattr(event.result, "exception_info", None) is not None + ) + await self._dispatch( + event.trial_name, + TrialEvent( + kind, + timestamp=event.timestamp.timestamp(), + payload=event.result if kind == TrialEventKind.END else None, + retry_predicted=retry_predicted, + ), + ) + except Exception as exc: + self._errors.append(f"hook {kind.value} for {event.trial_name}: {exc}") + if self.config.strict: + raise + + job.add_hook(harbor_event, callback) + + async def _dispatch(self, identity: str, event: TrialEvent) -> None: + if not accepts_trial_events(self._job_machine) and event.kind not in { + TrialEventKind.FINAL_RESULT, + TrialEventKind.SYNCED, + TrialEventKind.SYNC_FAILED, + TrialEventKind.OMIT, + }: + return + if identity not in self._trial_machines: + self._trial_machines[identity] = TrialMachine(identity) + self._trial_locks[identity] = asyncio.Lock() + async with self._trial_locks[identity]: + new_state, _effects = reduce_trial( + self._trial_machines[identity], + event, + strict=self.config.strict, + ) + self._trial_machines[identity] = new_state + + def _initialize(self, snapshot: JobSnapshot) -> RuntimeState: + previous = self._load_manifest(snapshot.job_dir) + self._manifest = previous + plan_by_trial = {plan.trial_name: plan for plan in snapshot.plans} + datasets: dict[str, DatasetBinding] = {} + source_tasks: dict[str, dict[str, Any]] = {} + for plan in snapshot.plans: + source_tasks.setdefault(plan.task.source, {})[plan.task.logical_key] = plan.task + + for source, task_map in source_tasks.items(): + scope = dataset_scope(source) + binding = DatasetBinding(scope) + datasets[scope] = binding + if self.config.dataset_mode != "sync": + continue + try: + if self.config.dataset_name and len(source_tasks) == 1: + name = self.config.dataset_name + else: + name = dataset_display_name(source, prefix=self.config.dataset_name or "harbor") + dataset = init_dataset( + project=self.config.project_name, + project_id=self.config.project_id, + name=name, + use_output=False, + metadata={"harbor": {"source": source, "scope": scope, "schema_version": _PLUGIN_VERSION}}, + ) + for task in task_map.values(): + normalized = normalize_json( + task.metadata, + max_bytes=self.config.max_custom_metadata_bytes, + redact_patterns=self.config.redact_patterns, + ) + self._warnings.extend(normalized.warnings) + dataset.insert( + id=dataset_record_id(scope, task.logical_key), + input=task.input, + expected=task.expected, + metadata=normalized.value, + ) + dataset.flush() + rows = list(dataset) + for row in rows: + if row.get("id") and row.get("_xact_id"): + binding.origins[row["id"]] = { + "object_type": "dataset", + "object_id": dataset.id, + "id": row["id"], + "created": row.get("created"), + "_xact_id": row["_xact_id"], + } + binding.dataset = dataset + except Exception as exc: + binding.error = str(exc) + self._warnings.append(f"dataset {scope} sync failed; continuing without association: {exc}") + + partitions: dict[str, Partition] = {} + partition_by_trial: dict[str, Partition] = {} + for plan in snapshot.plans: + scope = dataset_scope(plan.task.source) + semantic = semantic_agent_config( + plan.trial_config.agent, list(getattr(plan.trial_lock, "skills", []) or []) + ) + key = partition_key(scope, semantic) + partition = partitions.get(key) + if partition is None: + agent_name = ( + getattr(plan.trial_config.agent, "name", None) + or getattr(plan.trial_config.agent, "import_path", None) + or "agent" + ) + model = getattr(plan.trial_config.agent, "model_name", None) or "default" + prefix = self.config.experiment_prefix or snapshot.job_name + name = f"{prefix}-{snapshot.job_id[:8]} · {agent_name}@{model} · {plan.task.source} · {key[:8]}" + metadata = { + "harbor": { + "job_id": snapshot.job_id, + "job_name": snapshot.job_name, + "partition_key": key, + "semantic_agent_config": semantic, + } + } + dataset = datasets[scope].dataset + experiment = init( + project=self.config.project_name, + project_id=self.config.project_id, + experiment=name, + update=True, + dataset=dataset, + metadata=metadata, + base_experiment=self.config.base_experiment_name, + base_experiment_id=self.config.base_experiment_id, + ) + partition = Partition(key=key, name=name, dataset_scope=scope, experiment=experiment) + # Resolve lazy metadata now so initialization/auth failures are isolated. + partition.experiment_id = experiment.id + partitions[key] = partition + partition_by_trial[plan.trial_name] = partition + + return RuntimeState(snapshot, plan_by_trial, partition_by_trial, datasets, partitions) + + def _root_metadata(self, result: Any, plan: TrialPlan, machine: TrialMachine) -> dict[str, Any]: + raw_rewards = _rewards(result) + trial_custom = getattr(getattr(result, "agent_result", None), "metadata", None) or {} + normalized = normalize_json( + trial_custom if self.config.include_custom_metadata else {}, + max_bytes=self.config.max_custom_metadata_bytes, + redact_patterns=self.config.redact_patterns, + ) + task_custom = normalize_json( + plan.task.metadata.get("harbor", {}).get("custom", {}) if self.config.include_custom_metadata else {}, + max_bytes=self.config.max_custom_metadata_bytes, + redact_patterns=self.config.redact_patterns, + ) + self._warnings.extend((*normalized.warnings, *task_custom.warnings)) + error, traceback_value = _exception(result, self.config.include_tracebacks) + metadata: dict[str, Any] = { + "harbor": { + "job_id": self._runtime.snapshot.job_id if self._runtime else None, + "trial_id": str(result.id), + "task_name": result.task_name, + "agent": result.agent_info.name, + "model": result.agent_info.model_info.name if result.agent_info.model_info else None, + "attempt_index": plan.attempt_index, + "retry_index": machine.retry_index, + "raw_rewards": raw_rewards, + "custom": {"task": task_custom.value, "trial": normalized.value}, + "warnings": list(machine.warnings), + } + } + if error and traceback_value: + metadata["harbor"]["exception_traceback"] = traceback_value + return metadata + + def _start_phase( + self, + task_span: Any, + result: Any, + name: str, + timing_name: str, + trial_id: str, + root_start: float, + root_end: float, + **event: Any, + ) -> Any: + start, end = _timing(getattr(result, timing_name, None), root_start, root_end) + span = task_span.start_span( + name=name, + type="task", + id=child_span_id(trial_id, f"task/{name}"), + start_time=start, + set_current=False, + internal={"instrumentation": _INSTRUMENTATION}, + **event, + ) + span.end(end_time=end) + return span + + def _sync_final_result(self, result: Any) -> None: + if self._runtime is None: + raise RuntimeError("plugin is not initialized") + plan = self._runtime.plan_by_trial.get(result.trial_name) + partition = self._runtime.partition_by_trial.get(result.trial_name) + if plan is None or partition is None: + raise ValueError(f"final result {result.trial_name!r} is absent from the resolved plan") + machine = self._trial_machines[result.trial_name] + trial_id = str(result.id) + now = datetime.now().timestamp() + root_start = _seconds(getattr(result, "started_at", None), now) + root_end = _seconds(getattr(result, "finished_at", None), root_start) + if root_end < root_start: + root_end = root_start + error, _ = _exception(result, self.config.include_tracebacks) + metadata = self._root_metadata(result, plan, machine) + rewards = _rewards(result) + conversion = classify_rewards(rewards, self.config) + metadata["harbor"]["warnings"].extend(conversion.warnings) + if not rewards and error is None: + metadata["harbor"]["warnings"].append("trial has no reward and is unevaluated") + + binding = self._runtime.datasets[partition.dataset_scope] + record_id = dataset_record_id(partition.dataset_scope, plan.task.logical_key) + origin = binding.origins.get(record_id) + root_metrics = dict(conversion.metrics) + if machine.completed_attempts: + root_metrics["retries"] = max(machine.completed_attempts - 1, machine.retry_index) + root_event: dict[str, Any] = { + "id": trial_id, + "name": "eval", + "type": "eval", + "start_time": root_start, + "set_current": False, + "input": plan.task.input, + "expected": plan.task.expected, + "metadata": metadata, + "metrics": root_metrics, + } + if origin: + root_event["origin"] = origin + if error: + root_event["error"] = error + root = partition.experiment.start_span( + internal={"instrumentation": _INSTRUMENTATION}, + **root_event, + ) + task = root.start_span( + name="task", + type="task", + id=child_span_id(trial_id, "task"), + start_time=root_start, + set_current=False, + input=plan.task.input, + expected=plan.task.expected, + error=error, + internal={"instrumentation": _INSTRUMENTATION}, + ) + + self._start_phase(task, result, "environment_setup", "environment_setup", trial_id, root_start, root_end) + self._start_phase(task, result, "agent_setup", "agent_setup", trial_id, root_start, root_end) + agent_start, agent_end = _timing(getattr(result, "agent_execution", None), root_start, root_end) + execution_input: dict[str, Any] = {"task": plan.task.input} + extra_instructions: list[str] = [] + for path in getattr(result.config, "extra_instruction_paths", []) or []: + try: + extra_instructions.append(Path(path).read_text()) + except OSError: + continue + if extra_instructions: + execution_input["extra_instructions"] = extra_instructions + selected_artifacts, artifact_attachment_warnings = _artifact_attachments(result, self.config) + metadata["harbor"]["warnings"].extend(artifact_attachment_warnings) + agent_span = task.start_span( + name="agent_execution", + type="task", + id=child_span_id(trial_id, "task/agent_execution"), + start_time=agent_start, + set_current=False, + input=normalize_json( + execution_input, max_bytes=self.config.max_content_bytes, redact_patterns=self.config.redact_patterns + ).value, + internal={"instrumentation": _INSTRUMENTATION}, + ) + + atif_results: list[tuple[str | None, ATIFImportResult]] = [] + if self.config.trajectory_mode in {"atif", "summary"}: + for step_name, path in trajectory_paths(result): + if self.config.trajectory_mode == "summary": + imported = summarize_trajectory(path, self.config) + else: + prefix = "task/agent_execution" if step_name is None else f"task/step:{step_name}/agent_execution" + imported = import_trajectory( + agent_span, + path, + trial_id=trial_id, + semantic_prefix=prefix, + phase_start=agent_start, + phase_end=agent_end, + config=self.config, + ) + atif_results.append((step_name, imported)) + if selected_artifacts: + agent_span.log(output={"artifacts": selected_artifacts}) + agent_span.end(end_time=agent_end) + self._start_phase(task, result, "verification", "verifier", trial_id, root_start, root_end) + + for step in getattr(result, "step_results", None) or []: + step_start, step_end = _timing(getattr(step, "agent_execution", None), root_start, root_end) + step_span = task.start_span( + name=f"step:{step.step_name}", + type="task", + id=child_span_id(trial_id, f"task/step:{step.step_name}"), + start_time=step_start, + set_current=False, + internal={"instrumentation": _INSTRUMENTATION}, + ) + step_error, _ = _exception(step, self.config.include_tracebacks) + if step_error: + step_span.log(error=step_error) + step_span.end(end_time=step_end) + + trajectory_warnings = [warning for _, imported in atif_results for warning in imported.warnings] + repairs = [repair for _, imported in atif_results for repair in imported.repairs] + metadata["harbor"]["warnings"].extend(trajectory_warnings) + metadata["harbor"]["trajectory"] = { + # Report the mode so trajectory_mode="native", which deliberately skips + # ATIF because the agent is instrumented elsewhere, is distinguishable + # from a trajectory that could not be read. + "mode": self.config.trajectory_mode, + "present": bool(atif_results), + "schema_version": next( + (imported.schema_version for _, imported in atif_results if imported.schema_version), None + ), + "repairs": repairs, + } + # A multi-step trial has one trajectory per step, each with its own final + # message and its own aggregate token and cost totals. + raw_extra = _by_step( + [(name, imported.root_extra) for name, imported in atif_results if imported.root_extra], "trajectory" + ) + if raw_extra is not None and self.config.include_custom_metadata: + normalized_extra = normalize_json( + raw_extra, + max_bytes=self.config.max_custom_metadata_bytes, + redact_patterns=self.config.redact_patterns, + ) + metadata["harbor"]["trajectory"]["custom"] = normalized_extra.value + metadata["harbor"]["warnings"].extend(normalized_extra.warnings) + + output = _answer_from_metadata(result) + if output is None: + output = _by_step( + [ + (name, imported.final_message) + for name, imported in atif_results + if imported.final_message is not None + ], + "final", + keep_single_name=False, + ) + if output is None: + output = {"status": "completed" if error is None else "error"} + output = normalize_json( + output, max_bytes=self.config.max_content_bytes, redact_patterns=self.config.redact_patterns + ).value + if error is None: + task.log(output=output) + root.log(output=output) + task.log(metadata={"harbor": {"warnings": trajectory_warnings}}) + task.end(end_time=root_end) + + details_attachment, details_summary, detail_warnings = _attachment(reward_details_paths(result), self.config) + metadata["harbor"]["warnings"].extend(detail_warnings) + # The summary is the same for every score, so bound it once rather than + # re-normalizing a payload up to max_attachment_bytes per scorer span. + bounded_details = ( + None + if details_summary is None + else normalize_json( + details_summary, + max_bytes=self.config.max_content_bytes, + redact_patterns=self.config.redact_patterns, + ).value + ) + for score in conversion.scores: + scorer = root.start_span( + name=score.name, + type="score", + span_attributes={"purpose": "scorer"}, + id=child_span_id(trial_id, f"scorer/{score.source_key}"), + start_time=root_end, + set_current=False, + input={"reward": score.raw_value}, + internal={"instrumentation": _INSTRUMENTATION}, + ) + scorer_output: dict[str, Any] = {"score": score.value, "raw_reward": score.raw_value} + if bounded_details is not None: + scorer_output["reward_details_summary"] = bounded_details + if details_attachment is not None: + scorer_output["reward_details"] = details_attachment + scorer.log(output=scorer_output, scores={score.name: score.value}) + scorer.end(end_time=root_end) + + classifications: dict[str, list[dict[str, Any]]] = {} + for source_name, path in self.config.classifier_rules.items(): + classifier = root.start_span( + name=source_name, + type="classifier", + span_attributes={"purpose": "scorer"}, + id=child_span_id(trial_id, f"classifier/{source_name}"), + start_time=root_end, + set_current=False, + internal={"instrumentation": _INSTRUMENTATION}, + ) + try: + items = validate_classifications(extract_json_path(result, path)) + if items: + classifications[source_name] = items + classifier.log(output=items[0] if len(items) == 1 else items) + except Exception as exc: + classifier.log(error=f"invalid classifier {source_name}: {exc}") + metadata["harbor"]["warnings"].append(f"classifier {source_name!r} was malformed: {exc}") + classifier.end(end_time=root_end) + if classifications: + root.log(classifications=classifications) + + manifests, manifest_warnings = _read_json_summary( + artifact_manifest_paths(result), self.config.max_content_bytes + ) + metadata["harbor"]["warnings"].extend(manifest_warnings) + if manifests is not None: + metadata["harbor"]["artifact_manifest"] = normalize_json( + manifests, max_bytes=self.config.max_content_bytes + ).value + root.log(metadata=metadata) + root.end(end_time=root_end) + + def _persist_disabled_manifest(self) -> None: + if self._snapshot is None: + return + manifest = { + "manifest_version": _MANIFEST_VERSION, + "plugin_version": _PLUGIN_VERSION, + "job_id": self._snapshot.job_id, + "project": {"name": self.config.project_name, "id": self.config.project_id}, + "datasets": {}, + "experiments": {}, + "trials": {}, + "synced_trial_ids": [], + "warnings": self._warnings, + "errors": self._errors, + "disabled_reason": self._disabled_reason, + "completed": False, + } + path = self._snapshot.job_dir / "braintrust-sync.json" + temp_path = path.with_suffix(".json.tmp") + temp_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + os.replace(temp_path, path) + self._manifest = manifest + + @staticmethod + def _load_manifest(job_dir: Path) -> dict[str, Any]: + path = job_dir / "braintrust-sync.json" + try: + data = json.loads(path.read_text()) + return data if isinstance(data, dict) else {} + except (OSError, json.JSONDecodeError): + return {} + + def _persist_manifest(self, completed: bool) -> None: + if self._runtime is None: + return + snapshot = self._runtime.snapshot + manifest = { + "manifest_version": _MANIFEST_VERSION, + "plugin_version": _PLUGIN_VERSION, + "job_id": snapshot.job_id, + "project": {"name": self.config.project_name, "id": self.config.project_id}, + "datasets": { + scope: { + "id": binding.dataset.id if binding.dataset is not None else None, + "version": binding.dataset.version if binding.dataset is not None else None, + "error": binding.error, + } + for scope, binding in self._runtime.datasets.items() + }, + "experiments": { + key: {"id": partition.experiment_id, "name": partition.name} + for key, partition in self._runtime.partitions.items() + }, + "trials": { + name: { + "status": machine.status.value, + "retry_count": machine.retry_index, + "completed_attempts": machine.completed_attempts, + "warnings": list(machine.warnings), + } + for name, machine in self._trial_machines.items() + }, + "synced_trial_ids": sorted( + str(machine.final_result.id) + for machine in self._trial_machines.values() + if machine.status == TrialStatus.SYNCED and machine.final_result is not None + ), + "warnings": self._warnings, + "errors": self._errors, + "disabled_reason": self._disabled_reason, + "completed": completed, + } + path = snapshot.job_dir / "braintrust-sync.json" + temp_path = path.with_suffix(".json.tmp") + temp_path.write_text(json.dumps(manifest, indent=2, sort_keys=True, default=str) + "\n") + os.replace(temp_path, path) + self._manifest = manifest + + async def sync_job_directory(self, job_dir: str | Path) -> None: + """Backfill a persisted Harbor job directory with the online conversion core.""" + self._job_machine = reduce_job(self._job_machine, JobEvent.INITIALIZE, strict=self.config.strict) + snapshot, result = await asyncio.to_thread(load_backfill_snapshot, job_dir) + self._snapshot = snapshot + self._runtime = await asyncio.to_thread(self._initialize, snapshot) + for plan in snapshot.plans: + self._trial_machines[plan.trial_name] = TrialMachine(plan.trial_name) + self._trial_locks[plan.trial_name] = asyncio.Lock() + self._job_machine = reduce_job(self._job_machine, JobEvent.READY, strict=self.config.strict) + await self.on_job_end(result) + + +async def backfill_job(job_dir: str | Path, **plugin_options: Any) -> None: + """Backfill a Harbor job directory into Braintrust.""" + await HarborPlugin(**plugin_options).sync_job_directory(job_dir) diff --git a/py/src/braintrust/integrations/harbor/rewards.py b/py/src/braintrust/integrations/harbor/rewards.py new file mode 100644 index 00000000..d1895ee5 --- /dev/null +++ b/py/src/braintrust/integrations/harbor/rewards.py @@ -0,0 +1,152 @@ +"""Harbor reward and classifier conversion.""" + +import fnmatch +import math +from dataclasses import dataclass, field +from numbers import Real +from typing import Any + +from .config import PluginConfig + + +@dataclass(frozen=True) +class ScoreValue: + name: str + value: float + source_key: str + raw_value: int | float + transformed: bool = False + + +@dataclass(frozen=True) +class RewardConversion: + scores: tuple[ScoreValue, ...] = () + metrics: dict[str, int | float] = field(default_factory=dict) + warnings: tuple[str, ...] = () + + +def _numeric(value: Any) -> bool: + return isinstance(value, Real) and not isinstance(value, bool) and math.isfinite(float(value)) + + +def _matches(key: str, patterns: tuple[str, ...]) -> bool: + return any(fnmatch.fnmatchcase(key, pattern) for pattern in patterns) + + +def _metric_name(key: str) -> str: + # These keys have standardized Braintrust meanings. Harbor rewards are not + # presumed to share them, so retain an explicit integration namespace. + standard = { + "start", + "end", + "duration", + "tokens", + "prompt_tokens", + "completion_tokens", + "estimated_cost", + "time_to_first_token", + } + return f"harbor_reward.{key}" if key in standard else key + + +def classify_rewards(rewards: dict[str, Any] | None, config: PluginConfig) -> RewardConversion: + scores: list[ScoreValue] = [] + metrics: dict[str, int | float] = {} + warnings: list[str] = [] + if not rewards: + return RewardConversion() + + for key, raw in rewards.items(): + if not _numeric(raw): + warnings.append(f"reward {key!r} is not a finite number and was omitted") + continue + value = float(raw) + rule = config.reward_rules.get(key) + requested_type: str | None = rule.get("type") if rule else None + if requested_type is None: + if _matches(key, config.score_keys): + requested_type = "score" + elif _matches(key, config.metric_keys): + requested_type = "metric" + elif key == "reward" and 0 <= value <= 1: + requested_type = "score" + else: + requested_type = "metric" + + if requested_type == "metric": + metrics[_metric_name(key)] = raw + continue + + score_name = str(rule.get("score_name", key)) if rule else key + transformed = False + normalized = value + if rule and "min" in rule and "max" in rule: + minimum, maximum = float(rule["min"]), float(rule["max"]) + if minimum <= value <= maximum: + direction = rule.get("direction", "maximize") + normalized = (value - minimum) / (maximum - minimum) + if direction == "minimize": + normalized = (maximum - value) / (maximum - minimum) + transformed = True + else: + normalized = float("nan") + + if not math.isfinite(normalized) or not 0 <= normalized <= 1: + warning = f"configured score {key!r} has invalid value {raw!r}" + if config.invalid_score_policy == "error": + raise ValueError(warning) + warnings.append(warning) + if config.invalid_score_policy == "metric": + metrics[_metric_name(key)] = raw + continue + + scores.append( + ScoreValue( + name=score_name, + value=normalized, + source_key=key, + raw_value=raw, + transformed=transformed, + ) + ) + if transformed: + metrics[f"harbor_reward.raw.{key}"] = raw + + return RewardConversion(tuple(scores), metrics, tuple(warnings)) + + +def extract_json_path(value: Any, path: str) -> Any: + """Extract a documented dotted/JSON-pointer-like path from a model or mapping.""" + parts = [part for part in path.replace("/", ".").split(".") if part] + current = value + for part in parts: + if isinstance(current, dict): + if part not in current: + raise KeyError(path) + current = current[part] + elif isinstance(current, (list, tuple)): + current = current[int(part)] + else: + model_dump = getattr(current, "model_dump", None) + if callable(model_dump): + current = model_dump(mode="python", exclude_none=False) + if part not in current: + raise KeyError(path) + current = current[part] + else: + raise KeyError(path) + return current + + +def validate_classifications(value: Any) -> list[dict[str, Any]]: + items = value if isinstance(value, list) else [value] + validated: list[dict[str, Any]] = [] + for item in items: + if not isinstance(item, dict) or not isinstance(item.get("id"), str) or not item["id"]: + raise ValueError("classification items require a non-empty string id") + if "label" in item and item["label"] is not None and not isinstance(item["label"], str): + raise ValueError("classification item label must be a string or null") + if "metadata" in item and item["metadata"] is not None and not isinstance(item["metadata"], dict): + raise ValueError("classification item metadata must be a JSON object or null") + validated.append({key: item[key] for key in ("id", "label", "metadata") if key in item}) + return validated diff --git a/py/src/braintrust/integrations/harbor/state.py b/py/src/braintrust/integrations/harbor/state.py new file mode 100644 index 00000000..e96b6d52 --- /dev/null +++ b/py/src/braintrust/integrations/harbor/state.py @@ -0,0 +1,233 @@ +"""Pure reducer-based Harbor lifecycle state machines.""" + +from dataclasses import dataclass, replace +from enum import Enum +from typing import Any + + +class JobStatus(str, Enum): + NEW = "new" + INITIALIZING = "initializing" + ACTIVE = "active" + RECONCILING = "reconciling" + CLOSED = "closed" + DISABLED = "disabled" + FAILED = "failed" + + +class JobEvent(str, Enum): + INITIALIZE = "initialize" + READY = "ready" + RECONCILE = "reconcile" + CLOSE = "close" + DISABLE = "disable" + FAIL = "fail" + + +@dataclass(frozen=True) +class JobMachine: + status: JobStatus = JobStatus.NEW + warnings: tuple[str, ...] = () + + +def can_reconcile(state: JobMachine) -> bool: + """Report whether reconciliation may start, keeping the rule with the transitions. + + DISABLED and FAILED both reach RECONCILE illegally, so callers must ask rather + than enumerate terminal statuses at each site. + """ + return state.status == JobStatus.ACTIVE + + +def accepts_trial_events(state: JobMachine) -> bool: + return state.status in {JobStatus.ACTIVE, JobStatus.RECONCILING} + + +def reduce_job(state: JobMachine, event: JobEvent, *, strict: bool = False) -> JobMachine: + transitions = { + (JobStatus.NEW, JobEvent.INITIALIZE): JobStatus.INITIALIZING, + (JobStatus.INITIALIZING, JobEvent.READY): JobStatus.ACTIVE, + (JobStatus.ACTIVE, JobEvent.RECONCILE): JobStatus.RECONCILING, + (JobStatus.RECONCILING, JobEvent.CLOSE): JobStatus.CLOSED, + } + if event == JobEvent.DISABLE and state.status not in {JobStatus.CLOSED, JobStatus.FAILED}: + return replace(state, status=JobStatus.DISABLED) + if event == JobEvent.FAIL and state.status not in {JobStatus.CLOSED, JobStatus.DISABLED}: + return replace(state, status=JobStatus.FAILED) + target = transitions.get((state.status, event)) + if target is not None: + return replace(state, status=target) + message = f"illegal job transition {state.status.value} + {event.value}" + if strict: + raise ValueError(message) + return replace(state, warnings=(*state.warnings, message)) + + +class TrialStatus(str, Enum): + PENDING = "pending" + ACTIVE = "active" + WAITING_RETRY = "waiting_retry" + FINAL_CANDIDATE = "final_candidate" + CANCELLED = "cancelled" + FINALIZING = "finalizing" + SYNCED = "synced" + OMITTED = "omitted" + + +class TrialPhase(str, Enum): + STARTED = "started" + ENVIRONMENT = "environment" + AGENT = "agent" + AGENT_DONE = "agent_done" + VERIFICATION = "verification" + + +_PHASE_ORDER = { + TrialPhase.STARTED: 0, + TrialPhase.ENVIRONMENT: 1, + TrialPhase.AGENT: 2, + TrialPhase.AGENT_DONE: 3, + TrialPhase.VERIFICATION: 4, +} + + +class TrialEventKind(str, Enum): + START = "start" + ENVIRONMENT_START = "environment-start" + AGENT_START = "agent-start" + AGENT_END = "agent-end" + VERIFICATION_START = "verification-start" + END = "end" + CANCEL = "cancel" + FINAL_RESULT = "final_result" + OMIT = "omit" + SYNCED = "synced" + SYNC_FAILED = "sync_failed" + + +@dataclass(frozen=True) +class TrialEvent: + kind: TrialEventKind + timestamp: float | None = None + payload: Any = None + retry_predicted: bool = False + + +class EffectKind(str, Enum): + STAGE_RESULT = "stage_result" + RECORD_RETRY = "record_retry" + RECORD_CANCELLATION = "record_cancellation" + SYNC_FINAL = "sync_final" + CLOSE_OMITTED = "close_omitted" + + +@dataclass(frozen=True) +class Effect: + kind: EffectKind + payload: Any = None + + +@dataclass(frozen=True) +class TrialMachine: + identity: str + status: TrialStatus = TrialStatus.PENDING + phase: TrialPhase | None = None + retry_index: int = 0 + completed_attempts: int = 0 + warnings: tuple[str, ...] = () + final_result: Any = None + + +def _warn(state: TrialMachine, message: str, strict: bool) -> tuple[TrialMachine, tuple[Effect, ...]]: + if strict: + raise ValueError(message) + return replace(state, warnings=(*state.warnings, message)), () + + +def reduce_trial( + state: TrialMachine, + event: TrialEvent, + *, + strict: bool = False, +) -> tuple[TrialMachine, tuple[Effect, ...]]: + """Reduce one lifecycle event without performing I/O.""" + kind = event.kind + if kind == TrialEventKind.START: + if state.status == TrialStatus.ACTIVE: + return state, () + if state.status in { + TrialStatus.PENDING, + TrialStatus.WAITING_RETRY, + TrialStatus.FINAL_CANDIDATE, + TrialStatus.CANCELLED, + }: + retry_index = state.retry_index + effects: tuple[Effect, ...] = () + if state.status != TrialStatus.PENDING: + retry_index += 1 + effects = (Effect(EffectKind.RECORD_RETRY, retry_index),) + return replace( + state, status=TrialStatus.ACTIVE, phase=TrialPhase.STARTED, retry_index=retry_index + ), effects + return _warn(state, f"START after terminal state {state.status.value}", strict) + + phase_for_event = { + TrialEventKind.ENVIRONMENT_START: TrialPhase.ENVIRONMENT, + TrialEventKind.AGENT_START: TrialPhase.AGENT, + TrialEventKind.AGENT_END: TrialPhase.AGENT_DONE, + TrialEventKind.VERIFICATION_START: TrialPhase.VERIFICATION, + }.get(kind) + if phase_for_event is not None: + if state.status != TrialStatus.ACTIVE or state.phase is None: + return _warn(state, f"{kind.value} while {state.status.value}", strict) + current_order = _PHASE_ORDER[state.phase] + next_order = _PHASE_ORDER[phase_for_event] + if next_order == current_order: + return state, () + if next_order < current_order: + return _warn(state, f"backward phase {state.phase.value} -> {phase_for_event.value}", strict) + return replace(state, phase=phase_for_event), () + + if kind == TrialEventKind.END: + if state.status in {TrialStatus.WAITING_RETRY, TrialStatus.FINAL_CANDIDATE, TrialStatus.CANCELLED}: + return state, () + if state.status != TrialStatus.ACTIVE: + return _warn(state, f"END while {state.status.value}", strict) + status = TrialStatus.WAITING_RETRY if event.retry_predicted else TrialStatus.FINAL_CANDIDATE + return ( + replace(state, status=status, completed_attempts=state.completed_attempts + 1), + (Effect(EffectKind.STAGE_RESULT, event.payload),), + ) + + if kind == TrialEventKind.CANCEL: + if state.status == TrialStatus.CANCELLED: + return state, () + if state.status in {TrialStatus.SYNCED, TrialStatus.OMITTED}: + return _warn(state, f"CANCEL after terminal state {state.status.value}", strict) + return replace(state, status=TrialStatus.CANCELLED), (Effect(EffectKind.RECORD_CANCELLATION),) + + if kind == TrialEventKind.FINAL_RESULT: + if state.status == TrialStatus.SYNCED: + return state, () + if state.status == TrialStatus.OMITTED: + return _warn(state, "FINAL_RESULT after OMIT", strict) + return replace(state, status=TrialStatus.FINALIZING, final_result=event.payload), ( + Effect(EffectKind.SYNC_FINAL, event.payload), + ) + + if kind == TrialEventKind.SYNCED: + if state.status != TrialStatus.FINALIZING: + return _warn(state, f"SYNCED while {state.status.value}", strict) + return replace(state, status=TrialStatus.SYNCED), () + + if kind == TrialEventKind.SYNC_FAILED: + if state.status != TrialStatus.FINALIZING: + return _warn(state, f"SYNC_FAILED while {state.status.value}", strict) + return _warn(replace(state, status=TrialStatus.FINAL_CANDIDATE), str(event.payload), False) + + if kind == TrialEventKind.OMIT: + if state.status in {TrialStatus.SYNCED, TrialStatus.OMITTED}: + return state, () + return replace(state, status=TrialStatus.OMITTED), (Effect(EffectKind.CLOSE_OMITTED),) + + return _warn(state, f"unknown trial event {kind}", strict) diff --git a/py/src/braintrust/integrations/harbor/test_harbor.py b/py/src/braintrust/integrations/harbor/test_harbor.py new file mode 100644 index 00000000..cc22d035 --- /dev/null +++ b/py/src/braintrust/integrations/harbor/test_harbor.py @@ -0,0 +1,776 @@ +# Harbor is installed by the dedicated Python 3.12+ nox session, not by the +# cross-version pylint environment. +# pylint: disable=import-error + +import asyncio +import inspect +import json +import os +import re +from datetime import datetime, timezone +from pathlib import Path + +import pytest +from braintrust import flush, init +from braintrust.conftest import get_vcr_config +from braintrust.git_fields import GitMetadataSettings +from braintrust.integrations.harbor.atif import _usage_metrics, import_trajectory, summarize_trajectory +from braintrust.integrations.harbor.compat import artifact_manifest_paths, load_backfill_snapshot +from braintrust.integrations.harbor.config import PluginConfig +from braintrust.integrations.harbor.identity import ( + child_span_id, + dataset_record_id, + dataset_scope, + logical_task_key, + normalize_json, + partition_key, + semantic_agent_config, +) +from braintrust.integrations.harbor.plugin import ( + HarborPlugin, + RuntimeState, + _artifact_attachments, + _attachment, + _seconds, + _timing, +) +from braintrust.integrations.harbor.rewards import classify_rewards, validate_classifications +from braintrust.integrations.harbor.state import ( + JobEvent, + JobStatus, + TrialEvent, + TrialEventKind, + TrialMachine, + TrialPhase, + TrialStatus, + reduce_job, + reduce_trial, +) +from harbor.models.job.config import JobConfig, RetryConfig +from harbor.models.job.lock import AgentSkillLock, JobLock, TaskLock, TrialLock +from harbor.models.job.result import JobResult, JobStats +from harbor.models.task.id import LocalTaskId +from harbor.models.trajectories.trajectory import Trajectory +from harbor.models.trial.config import AgentConfig, EnvironmentConfig, TaskConfig, TrialConfig, VerifierConfig +from harbor.models.trial.result import AgentInfo, StepResult, TimingInfo, TrialResult + + +_ABSOLUTE_PATH_RE = re.compile(r"(?:/(?:Users|private|home)/[^\"\\\\\s]+|[A-Za-z]:\\\\[^\"\\\\\s]+)") + + +def _redact_cassette_body(body): + if not isinstance(body, (str, bytes)): + return body + is_bytes = isinstance(body, bytes) + text = body.decode("utf-8", errors="replace") if is_bytes else body + redacted = _ABSOLUTE_PATH_RE.sub("[REDACTED_PATH]", text) + return redacted.encode() if is_bytes else redacted + + +@pytest.fixture(scope="module") +def vcr_config(): + config = get_vcr_config() + scrub_response = config["before_record_response"] + + def before_record_request(request): + request.body = _redact_cassette_body(request.body) + return request + + def before_record_response(response): + response = scrub_response(response) + body = response.get("body", {}) + if "string" in body: + body["string"] = _redact_cassette_body(body["string"]) + return response + + return { + **config, + "before_record_request": before_record_request, + "before_record_response": before_record_response, + } + + +def test_config_environment_fallback_and_explicit_precedence(): + names = { + "HARBOR_BRAINTRUST_PROJECT": "harbor-project", + "HARBOR_BRAINTRUST_DATASET_MODE": "none", + "HARBOR_BRAINTRUST_SCORE_KEYS": '["reward", "correct*"]', + "HARBOR_BRAINTRUST_STRICT": "true", + } + original = {name: os.environ.get(name) for name in names} + try: + os.environ.update(names) + from braintrust.integrations.harbor import HarborPlugin + + config = HarborPlugin(strict=False).config + assert config.project_name == "harbor-project" + assert config.dataset_mode == "none" + assert config.score_keys == ("reward", "correct*") + assert config.strict is False + finally: + for name, value in original.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + +def test_harbor_resolves_the_plugin_through_its_entry_point(): + # Users select this plugin with `--plugin braintrust`, which Harbor resolves + # through the harbor.plugins entry-point group. Nothing else in the test suite + # exercises the packaging metadata, so a broken entry point would otherwise + # only surface as "plugin not found" for a real user. + from harbor.cli.plugin_registry import PLUGIN_ENTRY_POINT_GROUP, resolve_plugin_import_path + + assert PLUGIN_ENTRY_POINT_GROUP == "harbor.plugins" + assert resolve_plugin_import_path("braintrust") == "braintrust.integrations.harbor:HarborPlugin" + + +def test_plugin_implements_the_harbor_lifecycle_protocol(): + # The oldest supported Harbor release is pinned in [tool.braintrust.matrix.harbor]. + # AGENT_END is the API that sets that floor: it arrived in 0.16.0, and the + # lifecycle state machine subscribes to every event in this mapping, so a + # missing member disables the whole plugin at registration time. + from harbor.trial.hooks import TrialEvent as HarborTrialEvent + + for name in ("START", "ENVIRONMENT_START", "AGENT_START", "AGENT_END", "VERIFICATION_START", "END", "CANCEL"): + assert hasattr(HarborTrialEvent, name), name + + plugin = HarborPlugin(project_name="unused") + assert inspect.iscoroutinefunction(plugin.on_job_start) + assert inspect.iscoroutinefunction(plugin.on_job_end) + + +def test_config_rejects_overlapping_reward_patterns_and_invalid_bounds(): + with pytest.raises(ValueError, match="overlap"): + PluginConfig.from_options(score_keys=["correct*"], metric_keys=["correctness"]) + with pytest.raises(ValueError, match="min < max"): + PluginConfig.from_options(reward_rules={"latency": {"type": "score", "min": 1, "max": 1}}) + + +def test_reward_classification_is_semantic_not_range_based(): + config = PluginConfig.from_options( + reward_rules={ + "error_rate": { + "type": "score", + "direction": "minimize", + "min": 0, + "max": 10, + "score_name": "reliability", + } + }, + score_keys=["correctness"], + ) + result = classify_rewards( + {"reward": 0.8, "quality": 0.7, "correctness": 1, "error_rate": 2, "tokens": 40}, + config, + ) + + assert {score.name: score.value for score in result.scores} == { + "reward": 0.8, + "correctness": 1, + "reliability": 0.8, + } + assert result.metrics == { + "quality": 0.7, + "harbor_reward.raw.error_rate": 2, + "harbor_reward.tokens": 40, + } + + +def test_invalid_configured_score_defaults_to_metric(): + config = PluginConfig.from_options(score_keys=["raw"]) + result = classify_rewards({"raw": 5}, config) + assert result.scores == () + assert result.metrics == {"raw": 5} + assert result.warnings + + +def test_classification_validation_is_atomic_and_preserves_duplicates(): + items = validate_classifications( + [ + {"id": "cat", "label": "Cat", "metadata": {"confidence": "high"}}, + {"id": "cat", "label": None}, + ] + ) + assert items[0]["id"] == items[1]["id"] == "cat" + with pytest.raises(ValueError): + validate_classifications([{"id": "ok"}, {"label": "missing id"}]) + + +def test_metadata_normalization_redacts_secrets_paths_and_bounds_size(tmp_path): + normalized = normalize_json( + { + "api_key": "secret", + "nested": {"keep": None, "path": str(tmp_path / "private")}, + "message": "token=abc", + }, + max_bytes=1_000, + redact_patterns=(r"token=[a-z]+",), + ) + assert normalized.value["api_key"] == "[REDACTED]" + assert normalized.value["nested"]["keep"] is None + assert normalized.value["nested"]["path"] == "[REDACTED PATH]" + assert normalized.value["message"] == "[REDACTED]" + assert any("absolute path" in warning for warning in normalized.warnings) + assert normalized.complete is False + + +def test_secret_key_redaction_keeps_token_counters_and_credential_keys(): + normalized = normalize_json( + { + "max_tokens": 4096, + "total_tokens": 17, + "n_output_tokens": 3, + "usage": {"prompt_tokens": 10, "cached_tokens": 2}, + "api_key": "sk-live-value", + "github_token": "ghp-value", + "authorization": "Bearer value", + }, + max_bytes=10_000, + ) + + assert normalized.value["max_tokens"] == 4096 + assert normalized.value["total_tokens"] == 17 + assert normalized.value["n_output_tokens"] == 3 + assert normalized.value["usage"] == {"prompt_tokens": 10, "cached_tokens": 2} + assert normalized.value["api_key"] == "[REDACTED]" + assert normalized.value["github_token"] == "[REDACTED]" + assert normalized.value["authorization"] == "[REDACTED]" + assert any("sensitive key" in warning for warning in normalized.warnings) + + +def test_counting_keys_survive_redaction_even_when_their_value_is_not_numeric(): + # The numeric-value rule cannot cover these: AgentConfig.env is dict[str, str], + # so a token budget set through the environment always arrives as a string. + # Without the counter-segment exemption these template to ${MAX_TOKENS} and + # collapse agent configurations that differ only by their budget. + semantic = semantic_agent_config( + AgentConfig(name="agent", env={"MAX_TOKENS": "8000", "OPENAI_API_KEY": "sk-live-value"}), [] + ) + assert semantic["env"]["MAX_TOKENS"] == "8000" + assert semantic["env"]["OPENAI_API_KEY"] == "${OPENAI_API_KEY}" + + budgets = [ + semantic_agent_config(AgentConfig(name="agent", env={"MAX_TOKENS": value}), []) for value in ("1000", "8000") + ] + assert partition_key("scope", budgets[0]) != partition_key("scope", budgets[1]) + + # A container under a counting key must also be walked rather than collapsed. + walked = normalize_json({"token_usage": {"prompt_tokens": 1}}, max_bytes=10_000) + assert walked.value == {"token_usage": {"prompt_tokens": 1}} + assert walked.complete is True + + +def test_normalization_keeps_sandbox_paths(): + content = normalize_json( + {"path": "/app/answer.txt", "message": "wrote /app/answer.txt"}, + max_bytes=10_000, + redact_absolute_paths=False, + ) + assert content.value == {"path": "/app/answer.txt", "message": "wrote /app/answer.txt"} + assert content.warnings == () + assert content.complete is True + + +def test_size_bounding_accumulates_entry_sizes_instead_of_reserializing(): + # Bounding a container must not re-serialize every prefix: the payload is + # serialized again on its way to Braintrust, so entry sizes are accumulated. + # The selected entries must still match a byte-exact prefix walk, including + # escaping and multi-byte characters. + def canonical_bytes(value): + return len(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")) + + def reference(mapping, max_bytes): + kept = {} + for key in sorted(mapping): + if canonical_bytes({**kept, key: mapping[key]}) > max_bytes: + continue + kept[key] = mapping[key] + return kept + + payloads = [ + {f"key_{index}": {"body": "y" * 6, "n": index} for index in range(8)}, + {"é" * 4: "ü" * 6, "escaped": '\n\t"\\', "kept": 1, "emoji": "🙂" * 4}, + {"only": "x" * 30}, + ] + for payload in payloads: + # Sweep every budget so each entry boundary is crossed. A per-entry + # accounting error of even one byte changes the selected set somewhere in + # this range, which a handful of sampled budgets would miss. + # Start at 2: no dict fits a smaller budget, because "{}" is already two bytes. + for max_bytes in range(2, canonical_bytes(payload) + 2): + bounded = normalize_json(payload, max_bytes=max_bytes).value + assert isinstance(bounded, dict) + assert bounded == reference(payload, max_bytes), (payload, max_bytes) + assert canonical_bytes(bounded) <= max_bytes + + +def test_oversized_list_keeps_its_leading_elements_and_stays_a_list(): + items = [{"name": f"tool_{index}", "body": "x" * 100} for index in range(20)] + oversized = normalize_json(items, max_bytes=400) + + assert isinstance(oversized.value, list) + assert oversized.value == items[: len(oversized.value)] + assert 0 < len(oversized.value) < 20 + assert len(json.dumps(oversized.value, separators=(",", ":"))) <= 400 + assert oversized.complete is False + + +def test_dataset_scope_depends_only_on_the_task_source(): + # Scope must not encode the resolved task set: a narrower rerun or a backfill + # that cannot read one trial would otherwise fork a new dataset and experiment. + assert dataset_scope("suite") == "suite:tasks" + assert dataset_scope("suite") != dataset_scope("other-suite") + + +def test_config_rejects_unimplemented_retry_attempt_logging(): + with pytest.raises(ValueError, match="log_retry_attempts"): + PluginConfig.from_options(log_retry_attempts=True) + + +def test_ids_and_partition_are_deterministic_and_do_not_include_concurrency(tmp_path): + task_config = TrialConfig( + task=TaskConfig(path=Path("relative/task"), source="suite"), + agent=AgentConfig( + name="agent", + model_name="provider/model", + n_concurrent=1, + env={"API_KEY": "actual-secret", "MODE": "careful"}, + ), + ) + task_lock = TrialLock( + task=TaskLock(name="task", type="local", digest="sha256:" + "a" * 64, source="suite"), + agent=task_config.agent, + skills=[ + AgentSkillLock( + name="skill", + source=tmp_path / "skill", + digest="sha256:" + "b" * 64, + ) + ], + environment=EnvironmentConfig(), + verifier=VerifierConfig(), + ) + semantic = semantic_agent_config(task_config.agent, task_lock.skills) + key = logical_task_key(task_config, task_lock) + + assert "actual-secret" not in json.dumps(semantic) + assert semantic["env"]["API_KEY"] == "${API_KEY}" + assert partition_key(key, semantic) == partition_key(key, semantic) + assert dataset_record_id("scope", key) == dataset_record_id("scope", key) + assert child_span_id("trial", "task/verification") == child_span_id("trial", "task/verification") + + changed_concurrency = AgentConfig( + name="agent", + model_name="provider/model", + n_concurrent=99, + env={"API_KEY": "actual-secret", "MODE": "careful"}, + ) + assert semantic_agent_config(changed_concurrency, task_lock.skills) == semantic + + # A token budget is part of the agent's semantics, so it must partition. + small = semantic_agent_config(AgentConfig(name="agent", kwargs={"max_tokens": 1_000}), []) + large = semantic_agent_config(AgentConfig(name="agent", kwargs={"max_tokens": 8_000}), []) + assert small["kwargs"] == {"max_tokens": 1_000} + assert partition_key(key, small) != partition_key(key, large) + + +def test_harbor_naive_datetimes_use_the_host_timezone_and_timings_never_run_backward(): + started_at = datetime(2026, 7, 31, 9, 57, 7) + finished_at = datetime(2026, 7, 31, 9, 57, 41) + timing = TimingInfo(started_at=started_at, finished_at=finished_at) + + start, end = _timing(timing, 0, 0) + + assert start == started_at.timestamp() + assert end == finished_at.timestamp() + assert end >= start + assert _seconds(started_at, 0) == started_at.timestamp() + + backwards = TimingInfo(started_at=finished_at, finished_at=started_at) + backwards_start, backwards_end = _timing(backwards, 0, 0) + assert backwards_end == backwards_start + + +def _trial_result(trials_dir, trial_name, task_name, step_names=()): + """Build a real Harbor TrialResult rooted at a temporary trials directory.""" + return TrialResult( + task_name=task_name, + trial_name=trial_name, + trial_uri=f"file://{trials_dir / trial_name}", + task_id=LocalTaskId(path=Path("tasks") / task_name), + task_checksum="0" * 64, + config=TrialConfig( + task=TaskConfig(path=Path("tasks") / task_name, source="suite"), + agent=AgentConfig(name="agent", model_name="provider/model"), + trials_dir=trials_dir, + ), + agent_info=AgentInfo(name="agent", version="1.0.0"), + step_results=[StepResult(step_name=name) for name in step_names] or None, + ) + + +def test_artifact_attachments_are_scoped_per_step(tmp_path): + result = _trial_result(tmp_path, "trial-1", "task-a", step_names=("first", "second")) + for step_name, contents in (("first", b"first output"), ("second", b"second output")): + artifacts = tmp_path / "trial-1" / "steps" / step_name / "artifacts" + (artifacts / "logs").mkdir(parents=True) + (artifacts / "manifest.json").write_text("{}") + (artifacts / "logs" / "output.txt").write_bytes(contents) + + config = PluginConfig.from_options(attachments="all", artifact_include=["logs/*.txt"]) + attachments, warnings = _artifact_attachments(result, config) + + # Each step has its own artifacts root, so identically named files must not + # overwrite one another on the way to Braintrust. + assert sorted(attachments) == ["first/logs/output.txt", "second/logs/output.txt"] + assert attachments["first/logs/output.txt"].data == b"first output" + assert attachments["second/logs/output.txt"].data == b"second output" + assert warnings == [] + assert [step for step, _ in artifact_manifest_paths(result)] == ["first", "second"] + + +def test_reward_details_attachment_uses_the_per_file_limit(tmp_path): + # A multi-step trial merges one reward-details file per step, so the combined + # payload can exceed the per-file limit while every source file fits. + entries = [] + for step in ("first", "second"): + path = tmp_path / step / "reward-details.json" + path.parent.mkdir() + path.write_text(json.dumps({"criteria": "x" * 400})) + entries.append((step, path)) + + config = PluginConfig.from_options(max_attachment_bytes=600, max_total_attachment_bytes=100_000) + attachment, summary, warnings = _attachment(entries, config) + + assert attachment is None + # The summary keys each step's details, so a score cannot misattribute them. + assert sorted(summary) == ["first", "second"] + assert any("attachment size limit" in warning for warning in warnings) + + attachment, summary, warnings = _attachment([(None, entries[0][1])], config) + assert attachment is not None + assert attachment.reference["filename"] == "reward-details.json" + assert summary == {"criteria": "x" * 400} + assert warnings == [] + + +def test_disabled_plugin_does_not_reconcile_or_write_spans(): + plugin = HarborPlugin(project_name="unused") + # Reproduce the ordering that makes this reachable: the runtime is built, then + # a later step of on_job_start fails and disables the plugin. + plugin._job_machine = reduce_job(plugin._job_machine, JobEvent.INITIALIZE) + plugin._runtime = RuntimeState(snapshot=None, plan_by_trial={}, partition_by_trial={}, datasets={}, partitions={}) + plugin._disable("Braintrust initialization failed: boom") + job_result = JobResult( + id="00000000-0000-4000-8000-000000000001", + started_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + n_total_trials=1, + stats=JobStats(), + trial_results=[], + ) + + asyncio.run(plugin.on_job_end(job_result)) + + # A disabled plugin must not write an experiment its manifest reports as + # unsynchronized, and RECONCILE is not a legal transition out of DISABLED. + assert plugin._job_machine.status == JobStatus.DISABLED + assert plugin._job_machine.warnings == () + + +def test_backfill_matches_trial_locks_by_task_name(tmp_path): + trial_lock = TrialLock( + task=TaskLock(name="task-a", type="local", digest="sha256:" + "a" * 64, source="suite"), + agent=AgentConfig(name="agent", model_name="provider/model"), + environment=EnvironmentConfig(), + verifier=VerifierConfig(), + ) + (tmp_path / "config.json").write_text(JobConfig(job_name="job").model_dump_json()) + (tmp_path / "lock.json").write_text( + JobLock(n_concurrent_trials=1, retry=RetryConfig(), trials=[trial_lock]).model_dump_json() + ) + (tmp_path / "result.json").write_text( + JobResult( + id="00000000-0000-4000-8000-000000000002", + started_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + n_total_trials=2, + stats=JobStats(), + ).model_dump_json() + ) + for trial_name, task_name in (("trial-a", "task-a"), ("trial-b", "task-b")): + directory = tmp_path / trial_name + directory.mkdir() + (directory / "results.json").write_text(_trial_result(tmp_path, trial_name, task_name).model_dump_json()) + + snapshot, _ = load_backfill_snapshot(tmp_path) + locks = {plan.trial_name: plan.trial_lock for plan in snapshot.plans} + + assert locks["trial-a"] is not None + assert locks["trial-a"].task.name == "task-a" + # task-b has no lock entry. Falling back to another trial's lock would give it + # task-a's identity and skills, collapsing two tasks into one logical key. + assert locks["trial-b"] is None + keys = {plan.trial_name: plan.task.logical_key for plan in snapshot.plans} + assert keys["trial-a"] != keys["trial-b"] + + +def test_trial_reducer_retry_duplicate_backward_and_reconcile(): + state = TrialMachine("trial") + state, _ = reduce_trial(state, TrialEvent(TrialEventKind.START)) + assert state.status == TrialStatus.ACTIVE + assert state.phase == TrialPhase.STARTED + + duplicate, effects = reduce_trial(state, TrialEvent(TrialEventKind.START)) + assert duplicate == state + assert effects == () + + state, _ = reduce_trial(state, TrialEvent(TrialEventKind.AGENT_START)) + backward, _ = reduce_trial(state, TrialEvent(TrialEventKind.ENVIRONMENT_START)) + assert backward.phase == TrialPhase.AGENT + assert backward.warnings + + state, _ = reduce_trial(state, TrialEvent(TrialEventKind.END, retry_predicted=True)) + assert state.status == TrialStatus.WAITING_RETRY + assert state.completed_attempts == 1 + state, _ = reduce_trial(state, TrialEvent(TrialEventKind.START)) + assert state.retry_index == 1 + state, _ = reduce_trial(state, TrialEvent(TrialEventKind.END)) + assert state.status == TrialStatus.FINAL_CANDIDATE + + final = "authoritative-final-result" + state, effects = reduce_trial(state, TrialEvent(TrialEventKind.FINAL_RESULT, payload=final)) + assert state.status == TrialStatus.FINALIZING + assert effects[0].payload is final + state, _ = reduce_trial(state, TrialEvent(TrialEventKind.SYNCED)) + assert state.status == TrialStatus.SYNCED + + +def _import_into_experiment(trajectory_path, *, experiment_name, parent_id, trial_id, phase_end, config): + """Import a trajectory under a real agent_execution span in a cassette-backed experiment.""" + phase_start = datetime.fromisoformat("2026-01-01T00:00:00+00:00").timestamp() + experiment = init( + project="python-sdk-harbor-tests", + experiment=experiment_name, + update=True, + set_current=False, + git_metadata_settings=GitMetadataSettings(collect="none"), + api_key=os.environ.get("BRAINTRUST_API_KEY", "test-api-key-for-vcr-playback"), + ) + parent = experiment.start_span( + name="agent_execution", + type="task", + id=parent_id, + start_time=phase_start, + set_current=False, + ) + imported = import_trajectory( + parent, + trajectory_path, + trial_id=trial_id, + semantic_prefix="task/agent_execution", + phase_start=phase_start, + phase_end=phase_end, + config=config, + ) + parent.end(end_time=phase_end) + flush() + return experiment, imported + + +@pytest.mark.vcr +def test_atif_import_round_trips_with_real_sdks(tmp_path): + trajectory_path = tmp_path / "trajectory.json" + trajectory = Trajectory.model_validate( + { + "schema_version": "ATIF-v1.7", + "agent": { + "name": "terminus-2", + "version": "2.0.0", + "model_name": "openai/gpt-4o-mini", + "tool_definitions": [ + { + "type": "function", + "function": {"name": "calculator", "parameters": {"type": "object"}}, + } + ], + }, + "steps": [ + { + "step_id": 1, + "timestamp": "2026-01-01T00:00:00Z", + "source": "user", + "message": "What is 2+2?", + }, + { + "step_id": 2, + "timestamp": "2026-01-01T00:00:01Z", + "source": "agent", + "message": "I'll calculate it.", + "metrics": {"prompt_tokens": 10, "completion_tokens": 4, "cost_usd": 0.001}, + "tool_calls": [ + { + "tool_call_id": "call_1", + "function_name": "calculator", + "arguments": {"expression": "2+2"}, + } + ], + "observation": {"results": [{"source_call_id": "call_1", "content": "4"}]}, + }, + { + "step_id": 3, + "timestamp": "2026-01-01T00:00:02Z", + "source": "agent", + "message": "The answer is 4.", + "metrics": {"prompt_tokens": 15, "completion_tokens": 5}, + }, + ], + } + ) + trajectory_path.write_text(trajectory.model_dump_json()) + summary = summarize_trajectory(trajectory_path, PluginConfig.from_options()) + + assert summary.schema_version == "ATIF-v1.7" + assert summary.final_message == "The answer is 4." + assert summary.warnings == () + assert trajectory.steps[1].llm_call_count is None + assert _usage_metrics(trajectory.steps[1].metrics.model_dump(mode="python")) == { + "prompt_tokens": 10, + "completion_tokens": 4, + "tokens": 14, + "estimated_cost": 0.001, + } + + parent_id = "c7a87986-0192-5f40-9ac0-a535810f1fe7" + experiment, imported = _import_into_experiment( + trajectory_path, + experiment_name="harbor-atif-import", + parent_id=parent_id, + trial_id="trial-1", + phase_end=datetime.fromisoformat("2026-01-01T00:00:03+00:00").timestamp(), + config=PluginConfig.from_options(), + ) + + expected_ids = { + parent_id, + child_span_id("trial-1", "task/agent_execution/turn/2/llm"), + child_span_id("trial-1", "task/agent_execution/turn/2/tool/call_1"), + child_span_id("trial-1", "task/agent_execution/turn/3/llm"), + } + spans = [span for span in experiment if span["id"] in expected_ids] + leaves = sorted( + (span for span in spans if span["span_attributes"]["type"] in {"llm", "tool"}), + key=lambda span: (span["metrics"]["start"], span["span_attributes"]["exec_counter"]), + ) + + assert imported.imported_llm_spans == 2 + assert imported.imported_tool_spans == 1 + assert imported.repairs == ( + "step 2: inferred one model call from terminus-2 2.0.0 trajectory", + "step 3: inferred one model call from terminus-2 2.0.0 trajectory", + ) + assert [span["span_attributes"]["type"] for span in leaves] == ["llm", "tool", "llm"] + assert leaves[0]["metadata"] == { + "provider": "openai", + "model": "gpt-4o-mini", + "tools": [{"type": "function", "function": {"name": "calculator", "parameters": {"type": "object"}}}], + } + assert leaves[0]["metrics"]["tokens"] == 14 + assert leaves[1]["input"] == {"expression": "2+2"} + assert all( + span["context"]["span_origin"]["instrumentation"]["name"] == "braintrust.plugin.harbor" for span in leaves + ) + + +@pytest.mark.vcr +def test_atif_import_scopes_tool_calls_per_turn_and_reports_bounded_content(tmp_path): + trajectory_path = tmp_path / "trajectory.json" + trajectory = Trajectory.model_validate( + { + "schema_version": "ATIF-v1.7", + "agent": {"name": "generic-agent", "version": "1.0.0", "model_name": "openai/gpt-4o-mini"}, + "steps": [ + { + "step_id": 1, + "timestamp": "2026-01-01T00:00:00Z", + "source": "user", + "message": "Read /app/answer.txt and then /app/notes.txt.", + }, + { + "step_id": 2, + "timestamp": "2026-01-01T00:00:01Z", + "source": "agent", + "message": "Reading the answer file.", + "llm_call_count": 1, + "metrics": {"prompt_tokens": 12, "completion_tokens": 5}, + "tool_calls": [ + { + "tool_call_id": "call_1", + "function_name": "read_file", + "arguments": {"path": "/app/answer.txt"}, + } + ], + "observation": {"results": [{"source_call_id": "call_1", "content": "42"}]}, + }, + { + # ATIF only requires a tool_call_id to be unique within its + # step, so a producer may reuse call_1 in a later turn. + "step_id": 3, + "timestamp": "2026-01-01T00:00:02Z", + "source": "agent", + "message": "Now the notes.", + "llm_call_count": 1, + "metrics": {"prompt_tokens": 14, "completion_tokens": 6}, + "tool_calls": [ + { + "tool_call_id": "call_1", + "function_name": "read_file", + "arguments": {"path": "/app/notes.txt"}, + } + ], + "observation": {"results": [{"source_call_id": "call_1", "content": "none"}]}, + }, + { + "step_id": 4, + "timestamp": "2026-01-01T00:00:03Z", + "source": "agent", + "message": "y" * 600, + "llm_call_count": 1, + "metrics": {"prompt_tokens": 16, "completion_tokens": 7}, + }, + ], + } + ) + trajectory_path.write_text(trajectory.model_dump_json()) + + experiment, imported = _import_into_experiment( + trajectory_path, + experiment_name="harbor-atif-content-bounds", + parent_id="b4de2f31-7c05-5a9e-8d64-1f3a6c9b2e70", + trial_id="trial-2", + phase_end=datetime.fromisoformat("2026-01-01T00:00:04+00:00").timestamp(), + config=PluginConfig.from_options(max_content_bytes=300), + ) + + first_tool_id = child_span_id("trial-2", "task/agent_execution/turn/2/tool/call_1") + second_tool_id = child_span_id("trial-2", "task/agent_execution/turn/3/tool/call_1") + truncated_id = child_span_id("trial-2", "task/agent_execution/turn/4/summary") + assert first_tool_id != second_tool_id + + assert imported.imported_llm_spans == 2 + assert imported.imported_tool_spans == 2 + # Truncation and redaction must never be silent, and a truncated payload must + # not keep an llm label. + assert "step 4 message: truncated value: exceeded 300 bytes" in imported.warnings + assert "step 4: downgraded to task (message content was truncated or redacted)" in imported.warnings + + by_id = {span["id"]: span for span in experiment if span["id"] in {first_tool_id, second_tool_id, truncated_id}} + + # Sandbox paths are the substance of a filesystem tool call, and each turn's + # call must pair with the observation that actually answered it. + assert by_id[first_tool_id]["input"] == {"path": "/app/answer.txt"} + assert by_id[first_tool_id]["output"] == "42" + assert by_id[second_tool_id]["input"] == {"path": "/app/notes.txt"} + assert by_id[second_tool_id]["output"] == "none" + assert by_id[truncated_id]["span_attributes"]["type"] == "task" + assert len(by_id[truncated_id]["output"]["message"]) == 300