Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
191 changes: 188 additions & 3 deletions py/src/braintrust/integrations/cohere/test_cohere.py
Original file line number Diff line number Diff line change
Expand Up @@ -484,12 +484,14 @@ def test_wrap_cohere_chat_stream_v1_sync(memory_logger):

start = time.time()
events = []
for event in client.chat_stream(
stream = client.chat_stream(
model=CHAT_MODEL,
message="Say hi in one word.",
max_tokens=8,
):
)
for event in stream:
events.append(event)
stream.close()
end = time.time()

assert events
Expand Down Expand Up @@ -518,6 +520,38 @@ def test_wrap_cohere_chat_stream_v1_sync(memory_logger):
assert metrics.get("completion_tokens", 0) > 0


# Same request as the exhaustive test above; only the consumption differs.
@pytest.mark.vcr
@pytest.mark.parametrize("vcr_cassette_name", ["test_wrap_cohere_chat_stream_v1_sync"], ids=["shared"])
def test_wrap_cohere_chat_stream_v1_sync_close_finalizes_partially_read_stream(memory_logger, vcr_cassette_name):
assert not memory_logger.pop()
client = wrap_cohere(_v1_client())

stream = client.chat_stream(
model=CHAT_MODEL,
message="Say hi in one word.",
max_tokens=8,
)
first = next(stream)
assert first is not None
# Stop as soon as some text arrives, leaving the stream unfinished.
for event in stream:
if getattr(event, "event_type", None) == "text-generation":
break

stream.close()

# The explicit close finishes the span from the chunks read so far, without
# waiting for the stream to be exhausted.
spans = memory_logger.pop()
assert len(spans) == 1
span = spans[0]
assert span["span_attributes"]["name"] == "cohere.chat_stream"
assert not span.get("error")
assert span["metrics"]["start"] <= span["metrics"]["end"]
assert span["output"]


@pytest.mark.vcr
def test_wrap_cohere_chat_stream_v2_sync(memory_logger):
assert not memory_logger.pop()
Expand Down Expand Up @@ -574,9 +608,10 @@ def test_wrap_cohere_chat_stream_v2_sync_context_manager(memory_logger):
) as stream:
for event in stream:
events.append(event)
break
end = time.time()

assert events
assert len(events) == 1

spans = memory_logger.pop()
assert len(spans) == 1
Expand Down Expand Up @@ -778,3 +813,153 @@ def test_cohere_integration_setup_patches_audio_transcriptions(memory_logger, cl
class TestAutoInstrumentCohere:
def test_auto_instrument_cohere(self):
verify_autoinstrument_script("test_auto_cohere.py")


def test_wrap_cohere_async_chat_cancellation_records_error(memory_logger):
from unittest.mock import patch

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

lets avoid mocks/fakes! If we need to we should record new vcr cassettes

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed on avoiding hand-written fakes where a cassette can carry the behavior — the two stream-consumption tests already reuse the checked-in VCR cassette (test_wrap_cohere_chat_stream_v1_sync), and the subscriptions test uses a real client, so there is no stand-in client on those paths.

The remaining four cases cover behavior a cassette cannot express: a task cancelled mid-chat()/chat_stream(), and a stream whose close() raises. VCR replays recorded HTTP responses; there is no response to record when the caller raises CancelledError, and a failing close() is local cleanup rather than traffic. I used a narrow patch.object on the Cohere base client so the wrapper's span-finalization code still runs — the instrumentation itself is not replaced.

I am happy to change the approach if you have a preferred pattern, e.g. cancelling a real VCR-backed client task at a specific await point, or using a fault-injecting transport fixture. Which do you prefer?


from cohere.base_client import AsyncBaseCohere

assert not memory_logger.pop()

async def _cancelled_chat(*args, **kwargs):
raise asyncio.CancelledError("request was cancelled")

async def _run():
with patch.object(AsyncBaseCohere, "chat", _cancelled_chat):
client = wrap_cohere(_v1_async_client())
await client.chat(model=CHAT_MODEL, message="hello")

with pytest.raises(asyncio.CancelledError):
asyncio.run(_run())

spans = memory_logger.pop()
assert len(spans) == 1
assert "request was cancelled" in spans[0].get("error", "")
assert spans[0]["metrics"]["end"] >= spans[0]["metrics"]["start"]


def test_wrap_cohere_async_chat_message_less_cancellation_records_error(memory_logger):

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Parameterize the duplicate async cancellation cases

This new case duplicates test_wrap_cohere_async_chat_cancellation_records_error: the client setup, patch, cancellation propagation, span-count check, and end-time check are identical; only the exception message and expected diagnostic differ. AGENTS.md mandates the integration skill's review checks, whose “Consolidate before handoff” rule requires merging input-only variants into a parametrized test. Parameterize the cancellation exception and expected diagnostic, retaining the message-less regression as a parameter, so the shared setup and finalization assertions cannot drift between copies.

AGENTS.md reference: AGENTS.md:L136-L136

Useful? React with 👍 / 👎.

from unittest.mock import patch

from cohere.base_client import AsyncBaseCohere

assert not memory_logger.pop()

async def _cancelled_chat(*args, **kwargs):
raise asyncio.CancelledError

async def _run():
with patch.object(AsyncBaseCohere, "chat", _cancelled_chat):
client = wrap_cohere(_v1_async_client())
await client.chat(model=CHAT_MODEL, message="hello")

with pytest.raises(asyncio.CancelledError):
asyncio.run(_run())

spans = memory_logger.pop()
assert len(spans) == 1
# A cancellation raised without a message must still name what ended the span.
assert spans[0].get("error") == "CancelledError"
assert spans[0]["metrics"]["end"] >= spans[0]["metrics"]["start"]


def test_wrap_cohere_async_chat_stream_cancellation_records_error(memory_logger):
from unittest.mock import patch

from cohere.base_client import AsyncBaseCohere

assert not memory_logger.pop()

class _MockStream:
def __aiter__(self):
return self

async def __anext__(self):
raise asyncio.CancelledError("stream was cancelled")

async def _run():
with patch.object(AsyncBaseCohere, "chat_stream", lambda *a, **kw: _MockStream()):
client = wrap_cohere(_v1_async_client())
stream = await client.chat_stream(model=CHAT_MODEL, message="hello")
async for _ in stream:
pass

with pytest.raises(asyncio.CancelledError):
asyncio.run(_run())

spans = memory_logger.pop()
assert len(spans) == 1
assert "stream was cancelled" in spans[0].get("error", "")
assert spans[0]["metrics"]["end"] >= spans[0]["metrics"]["start"]


def test_wrap_cohere_chat_stream_close_error_records_error(memory_logger):
from unittest.mock import patch

from cohere.base_client import BaseCohere

assert not memory_logger.pop()

class _MockSyncStream:
def __init__(self):
self.custom_attr = "custom_value"

def __iter__(self):
return self

def __next__(self):
return {"event_type": "text-generation", "text": "hello"}

def close(self):
raise RuntimeError("close failed")

with patch.object(BaseCohere, "chat_stream", lambda *a, **kw: _MockSyncStream()):

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Use real Cohere streams for the cleanup regression coverage

For normal stream attribute/cleanup behavior, this replaces the provider with hand-written stand-ins, contrary to AGENTS.md's instruction to use real library classes and drive them with cassettes. The sync stand-in isn't even an iterator (__next__ is missing), yet the test passes because no chunks are read, so it cannot establish that closing a real Cohere stream preserves iteration and finalizes tracing. Extend the existing cassette-backed streaming tests to cover partial consumption and explicit closure instead.

AGENTS.md reference: AGENTS.md:L34-L35

Useful? React with 👍 / 👎.

client = wrap_cohere(_v1_client())
stream = client.chat_stream(model=CHAT_MODEL, message="hello")
assert stream.custom_attr == "custom_value"
next(stream)
with pytest.raises(RuntimeError, match="close failed"):
stream.close()

spans = memory_logger.pop()
assert len(spans) == 1
assert "close failed" in spans[0].get("error", "")
assert spans[0]["metrics"]["end"] >= spans[0]["metrics"]["start"]


def test_wrap_cohere_async_chat_stream_aclose_error_records_error(memory_logger):
from unittest.mock import patch

from cohere.base_client import AsyncBaseCohere

assert not memory_logger.pop()

class _MockAsyncStream:
def __init__(self):
self.custom_async_attr = "async_value"

def __aiter__(self):
return self

async def __anext__(self):
return {"event_type": "text-generation", "text": "hello"}

async def aclose(self):
raise asyncio.CancelledError("aclose was cancelled")

async def _run():
with patch.object(AsyncBaseCohere, "chat_stream", lambda *a, **kw: _MockAsyncStream()):
client = wrap_cohere(_v1_async_client())
stream = await client.chat_stream(model=CHAT_MODEL, message="hello")
assert stream.custom_async_attr == "async_value"
await stream.__anext__()
await stream.aclose()

with pytest.raises(asyncio.CancelledError):
asyncio.run(_run())

spans = memory_logger.pop()
assert len(spans) == 1
assert "aclose was cancelled" in spans[0].get("error", "")
assert spans[0]["metrics"]["end"] >= spans[0]["metrics"]["start"]
61 changes: 54 additions & 7 deletions py/src/braintrust/integrations/cohere/tracing.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
derived from ``usage`` / ``meta.billed_units`` / ``meta.tokens``.
"""

import asyncio
import logging
import time
from typing import Any
Expand Down Expand Up @@ -696,7 +697,7 @@ async def _trace_async_call(name: str, span_input: Any, metadata: dict[str, Any]
start_time = time.time()
try:
result = await wrapped(*args, **kwargs)
except Exception as error:
except (Exception, asyncio.CancelledError) as error:
_log_error_and_end_span(span, error)
Comment on lines +700 to 701

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Preserve a nonempty diagnostic for ordinary task cancellation

When a request is cancelled with Task.cancel() or by asyncio.wait_for(), the resulting CancelledError normally has no message. _log_error_and_end_span passes that exception directly to span.log, whose bt_safe_deep_copy serialization falls back to str(error). An isolated reproduction of this path exports {"error": ""} and ends the span, losing any indication that cancellation caused the termination. The newly caught streaming and cleanup cancellations have the same problem; the added tests miss it because every injected cancellation supplies a message. Log a formatted exception or fall back to its type name, and cover message-less cancellation.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 06f991e. _log_error_and_end_span now falls back to type(error).__name__ when str(error) is empty, so a bare CancelledError from Task.cancel() exports error: "CancelledError" instead of "". The change is in the shared helper, so the streaming and cleanup cancellation paths you flagged are covered by the same fix.

test_wrap_cohere_async_chat_message_less_cancellation_records_error raises asyncio.CancelledError with no message and asserts the exported error is "CancelledError"; it fails without the fallback.

Verified: pytest src/braintrust/integrations/test_utils.py src/braintrust/integrations/cohere/test_cohere.py --vcr-record=none → 78 passed.

raise

Expand Down Expand Up @@ -794,7 +795,7 @@ async def _async_chat_stream_wrapper(wrapped, instance, args, kwargs): # noqa:
start_time = time.time()
try:
iterator = wrapped(*args, **kwargs)
except Exception as error:
except (Exception, asyncio.CancelledError) as error:
_log_error_and_end_span(span, error)
raise

Expand All @@ -803,7 +804,7 @@ async def _async_chat_stream_wrapper(wrapped, instance, args, kwargs): # noqa:
if hasattr(iterator, "__await__"):
try:
iterator = await iterator
except Exception as error:
except (Exception, asyncio.CancelledError) as error:
_log_error_and_end_span(span, error)
raise

Expand All @@ -824,6 +825,9 @@ def __init__(self, iterator: Any, span: Any, start_time: float):
self._chunks: list[Any] = []
self._finished = False

def __getattr__(self, name: str) -> Any:
return getattr(self._iterator, name)

def _record(self, event: Any) -> None:
if self._first_token_time is None:
self._first_token_time = time.time()
Expand Down Expand Up @@ -867,7 +871,11 @@ def __exit__(self, exc_type, exc_value, traceback):
context_manager = getattr(self, "_context_manager", self._iterator)
exit_method = getattr(context_manager, "__exit__", None)
if exit_method is not None:
suppress = bool(exit_method(exc_type, exc_value, traceback))
try:
suppress = bool(exit_method(exc_type, exc_value, traceback))
except (Exception, asyncio.CancelledError) as error:
self._finish(error=error)
raise
if exc_value is not None:
self._finish(error=exc_value)
else:
Expand All @@ -883,12 +891,23 @@ def __next__(self):
except StopIteration:
self._finish()
raise
except Exception as error:
except (Exception, asyncio.CancelledError) as error:
self._finish(error=error)
raise
self._record(event)
return event

def close(self) -> None:
close_method = getattr(self._iterator, "close", None)
try:
if callable(close_method):
close_method()
except (Exception, asyncio.CancelledError) as error:
self._finish(error=error)
raise
else:
self._finish()


class _AsyncTracedChatStream(_ChatStreamTracker):
"""Async counterpart of :class:`_TracedChatStream`."""
Expand All @@ -906,7 +925,11 @@ async def __aexit__(self, exc_type, exc_value, traceback):
context_manager = getattr(self, "_context_manager", self._iterator)
aexit = getattr(context_manager, "__aexit__", None)
if aexit is not None:
suppress = bool(await aexit(exc_type, exc_value, traceback))
try:
suppress = bool(await aexit(exc_type, exc_value, traceback))
except (Exception, asyncio.CancelledError) as error:
self._finish(error=error)
raise
if exc_value is not None:
self._finish(error=exc_value)
else:
Expand All @@ -922,12 +945,36 @@ async def __anext__(self):
except StopAsyncIteration:
self._finish()
raise
except Exception as error:
except (Exception, asyncio.CancelledError) as error:
self._finish(error=error)
raise
self._record(event)
return event

async def aclose(self) -> None:
aclose_method = getattr(self._iterator, "aclose", None)
try:
if callable(aclose_method):
res = aclose_method()
if hasattr(res, "__await__"):
await res
except (Exception, asyncio.CancelledError) as error:
self._finish(error=error)
raise
else:
self._finish()

def close(self) -> None:
close_method = getattr(self._iterator, "close", None)
try:
if callable(close_method):
close_method()
except (Exception, asyncio.CancelledError) as error:
self._finish(error=error)
raise
else:
self._finish()


# ---------------------------------------------------------------------------
# Manual wrapping — `wrap_cohere(client)`
Expand Down
15 changes: 15 additions & 0 deletions py/src/braintrust/integrations/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -548,6 +548,21 @@ def test_log_error_and_end_span_logs_error_then_ends():
span.end.assert_called_once_with()


def test_log_error_and_end_span_survives_an_unprintable_error():
class Unprintable(Exception):
def __str__(self):
raise RuntimeError("cannot render")

span = unittest.mock.Mock()

_log_error_and_end_span(span, Unprintable())

# The probe must not escape before the span is ended; the type name still says
# what ended it.
span.log.assert_called_once_with(error="Unprintable")
span.end.assert_called_once_with()


def test_merge_timing_and_usage_metrics(monkeypatch):
monkeypatch.setattr("braintrust.integrations.utils.time.time", lambda: 15.0)

Expand Down
16 changes: 14 additions & 2 deletions py/src/braintrust/integrations/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -821,6 +821,18 @@ def _log_and_end_span(


def _log_error_and_end_span(span: Span, error: BaseException) -> None:
"""Log an error to *span* and immediately end it."""
span.log(error=error)
"""Log an error to *span* and immediately end it.

``str()`` of an exception raised without a message — a bare ``CancelledError``
from ``Task.cancel()``, for example — is empty, which would export an empty
``error`` field. Fall back to the type name so the span still records what
ended it. ``str()`` can also raise, for an exception with a broken
``__str__``, and that must not stop the span from ending: the probe is
contained and the type name is used instead.
"""
try:
message = str(error)
except Exception:
message = ""
span.log(error=error if message else type(error).__name__)
span.end()