Repository navigation
fix(cohere): handle cancellation and forward stream attributes #1160
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
9aabf83
0f4d2fa
06f991e
2b7f7ec
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -484,12 +484,14 @@ def test_wrap_cohere_chat_stream_v1_sync(memory_logger): | |
|
|
||
| start = time.time() | ||
| events = [] | ||
| for event in client.chat_stream( | ||
| stream = client.chat_stream( | ||
| model=CHAT_MODEL, | ||
| message="Say hi in one word.", | ||
| max_tokens=8, | ||
| ): | ||
| ) | ||
| for event in stream: | ||
| events.append(event) | ||
| stream.close() | ||
| end = time.time() | ||
|
|
||
| assert events | ||
|
|
@@ -518,6 +520,38 @@ def test_wrap_cohere_chat_stream_v1_sync(memory_logger): | |
| assert metrics.get("completion_tokens", 0) > 0 | ||
|
|
||
|
|
||
| # Same request as the exhaustive test above; only the consumption differs. | ||
| @pytest.mark.vcr | ||
| @pytest.mark.parametrize("vcr_cassette_name", ["test_wrap_cohere_chat_stream_v1_sync"], ids=["shared"]) | ||
| def test_wrap_cohere_chat_stream_v1_sync_close_finalizes_partially_read_stream(memory_logger, vcr_cassette_name): | ||
| assert not memory_logger.pop() | ||
| client = wrap_cohere(_v1_client()) | ||
|
|
||
| stream = client.chat_stream( | ||
| model=CHAT_MODEL, | ||
| message="Say hi in one word.", | ||
| max_tokens=8, | ||
| ) | ||
| first = next(stream) | ||
| assert first is not None | ||
| # Stop as soon as some text arrives, leaving the stream unfinished. | ||
| for event in stream: | ||
| if getattr(event, "event_type", None) == "text-generation": | ||
| break | ||
|
|
||
| stream.close() | ||
|
|
||
| # The explicit close finishes the span from the chunks read so far, without | ||
| # waiting for the stream to be exhausted. | ||
| spans = memory_logger.pop() | ||
| assert len(spans) == 1 | ||
| span = spans[0] | ||
| assert span["span_attributes"]["name"] == "cohere.chat_stream" | ||
| assert not span.get("error") | ||
| assert span["metrics"]["start"] <= span["metrics"]["end"] | ||
| assert span["output"] | ||
|
|
||
|
|
||
| @pytest.mark.vcr | ||
| def test_wrap_cohere_chat_stream_v2_sync(memory_logger): | ||
| assert not memory_logger.pop() | ||
|
|
@@ -574,9 +608,10 @@ def test_wrap_cohere_chat_stream_v2_sync_context_manager(memory_logger): | |
| ) as stream: | ||
| for event in stream: | ||
| events.append(event) | ||
| break | ||
| end = time.time() | ||
|
|
||
| assert events | ||
| assert len(events) == 1 | ||
|
|
||
| spans = memory_logger.pop() | ||
| assert len(spans) == 1 | ||
|
|
@@ -778,3 +813,153 @@ def test_cohere_integration_setup_patches_audio_transcriptions(memory_logger, cl | |
| class TestAutoInstrumentCohere: | ||
| def test_auto_instrument_cohere(self): | ||
| verify_autoinstrument_script("test_auto_cohere.py") | ||
|
|
||
|
|
||
| def test_wrap_cohere_async_chat_cancellation_records_error(memory_logger): | ||
| from unittest.mock import patch | ||
|
|
||
| from cohere.base_client import AsyncBaseCohere | ||
|
|
||
| assert not memory_logger.pop() | ||
|
|
||
| async def _cancelled_chat(*args, **kwargs): | ||
| raise asyncio.CancelledError("request was cancelled") | ||
|
|
||
| async def _run(): | ||
| with patch.object(AsyncBaseCohere, "chat", _cancelled_chat): | ||
| client = wrap_cohere(_v1_async_client()) | ||
| await client.chat(model=CHAT_MODEL, message="hello") | ||
|
|
||
| with pytest.raises(asyncio.CancelledError): | ||
| asyncio.run(_run()) | ||
|
|
||
| spans = memory_logger.pop() | ||
| assert len(spans) == 1 | ||
| assert "request was cancelled" in spans[0].get("error", "") | ||
| assert spans[0]["metrics"]["end"] >= spans[0]["metrics"]["start"] | ||
|
|
||
|
|
||
| def test_wrap_cohere_async_chat_message_less_cancellation_records_error(memory_logger): | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
This new case duplicates AGENTS.md reference: AGENTS.md:L136-L136 Useful? React with 👍 / 👎. |
||
| from unittest.mock import patch | ||
|
|
||
| from cohere.base_client import AsyncBaseCohere | ||
|
|
||
| assert not memory_logger.pop() | ||
|
|
||
| async def _cancelled_chat(*args, **kwargs): | ||
| raise asyncio.CancelledError | ||
|
|
||
| async def _run(): | ||
| with patch.object(AsyncBaseCohere, "chat", _cancelled_chat): | ||
| client = wrap_cohere(_v1_async_client()) | ||
| await client.chat(model=CHAT_MODEL, message="hello") | ||
|
|
||
| with pytest.raises(asyncio.CancelledError): | ||
| asyncio.run(_run()) | ||
|
|
||
| spans = memory_logger.pop() | ||
| assert len(spans) == 1 | ||
| # A cancellation raised without a message must still name what ended the span. | ||
| assert spans[0].get("error") == "CancelledError" | ||
| assert spans[0]["metrics"]["end"] >= spans[0]["metrics"]["start"] | ||
|
|
||
|
|
||
| def test_wrap_cohere_async_chat_stream_cancellation_records_error(memory_logger): | ||
| from unittest.mock import patch | ||
|
|
||
| from cohere.base_client import AsyncBaseCohere | ||
|
|
||
| assert not memory_logger.pop() | ||
|
|
||
| class _MockStream: | ||
| def __aiter__(self): | ||
| return self | ||
|
|
||
| async def __anext__(self): | ||
| raise asyncio.CancelledError("stream was cancelled") | ||
|
|
||
| async def _run(): | ||
| with patch.object(AsyncBaseCohere, "chat_stream", lambda *a, **kw: _MockStream()): | ||
| client = wrap_cohere(_v1_async_client()) | ||
| stream = await client.chat_stream(model=CHAT_MODEL, message="hello") | ||
| async for _ in stream: | ||
| pass | ||
|
|
||
| with pytest.raises(asyncio.CancelledError): | ||
| asyncio.run(_run()) | ||
|
|
||
| spans = memory_logger.pop() | ||
| assert len(spans) == 1 | ||
| assert "stream was cancelled" in spans[0].get("error", "") | ||
| assert spans[0]["metrics"]["end"] >= spans[0]["metrics"]["start"] | ||
|
|
||
|
|
||
| def test_wrap_cohere_chat_stream_close_error_records_error(memory_logger): | ||
| from unittest.mock import patch | ||
|
|
||
| from cohere.base_client import BaseCohere | ||
|
|
||
| assert not memory_logger.pop() | ||
|
|
||
| class _MockSyncStream: | ||
| def __init__(self): | ||
| self.custom_attr = "custom_value" | ||
|
|
||
| def __iter__(self): | ||
| return self | ||
|
|
||
| def __next__(self): | ||
| return {"event_type": "text-generation", "text": "hello"} | ||
|
|
||
| def close(self): | ||
| raise RuntimeError("close failed") | ||
|
|
||
| with patch.object(BaseCohere, "chat_stream", lambda *a, **kw: _MockSyncStream()): | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
For normal stream attribute/cleanup behavior, this replaces the provider with hand-written stand-ins, contrary to AGENTS.md reference: AGENTS.md:L34-L35 Useful? React with 👍 / 👎. |
||
| client = wrap_cohere(_v1_client()) | ||
| stream = client.chat_stream(model=CHAT_MODEL, message="hello") | ||
| assert stream.custom_attr == "custom_value" | ||
| next(stream) | ||
| with pytest.raises(RuntimeError, match="close failed"): | ||
| stream.close() | ||
|
|
||
| spans = memory_logger.pop() | ||
| assert len(spans) == 1 | ||
| assert "close failed" in spans[0].get("error", "") | ||
| assert spans[0]["metrics"]["end"] >= spans[0]["metrics"]["start"] | ||
|
|
||
|
|
||
| def test_wrap_cohere_async_chat_stream_aclose_error_records_error(memory_logger): | ||
| from unittest.mock import patch | ||
|
|
||
| from cohere.base_client import AsyncBaseCohere | ||
|
|
||
| assert not memory_logger.pop() | ||
|
|
||
| class _MockAsyncStream: | ||
| def __init__(self): | ||
| self.custom_async_attr = "async_value" | ||
|
|
||
| def __aiter__(self): | ||
| return self | ||
|
|
||
| async def __anext__(self): | ||
| return {"event_type": "text-generation", "text": "hello"} | ||
|
|
||
| async def aclose(self): | ||
| raise asyncio.CancelledError("aclose was cancelled") | ||
|
|
||
| async def _run(): | ||
| with patch.object(AsyncBaseCohere, "chat_stream", lambda *a, **kw: _MockAsyncStream()): | ||
| client = wrap_cohere(_v1_async_client()) | ||
| stream = await client.chat_stream(model=CHAT_MODEL, message="hello") | ||
| assert stream.custom_async_attr == "async_value" | ||
| await stream.__anext__() | ||
| await stream.aclose() | ||
|
|
||
| with pytest.raises(asyncio.CancelledError): | ||
| asyncio.run(_run()) | ||
|
|
||
| spans = memory_logger.pop() | ||
| assert len(spans) == 1 | ||
| assert "aclose was cancelled" in spans[0].get("error", "") | ||
| assert spans[0]["metrics"]["end"] >= spans[0]["metrics"]["start"] | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,6 +10,7 @@ | |
| derived from ``usage`` / ``meta.billed_units`` / ``meta.tokens``. | ||
| """ | ||
|
|
||
| import asyncio | ||
| import logging | ||
| import time | ||
| from typing import Any | ||
|
|
@@ -696,7 +697,7 @@ async def _trace_async_call(name: str, span_input: Any, metadata: dict[str, Any] | |
| start_time = time.time() | ||
| try: | ||
| result = await wrapped(*args, **kwargs) | ||
| except Exception as error: | ||
| except (Exception, asyncio.CancelledError) as error: | ||
| _log_error_and_end_span(span, error) | ||
|
Comment on lines
+700
to
701
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When a request is cancelled with Useful? React with 👍 / 👎.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed in 06f991e.
Verified: |
||
| raise | ||
|
|
||
|
|
@@ -794,7 +795,7 @@ async def _async_chat_stream_wrapper(wrapped, instance, args, kwargs): # noqa: | |
| start_time = time.time() | ||
| try: | ||
| iterator = wrapped(*args, **kwargs) | ||
| except Exception as error: | ||
| except (Exception, asyncio.CancelledError) as error: | ||
| _log_error_and_end_span(span, error) | ||
| raise | ||
|
|
||
|
|
@@ -803,7 +804,7 @@ async def _async_chat_stream_wrapper(wrapped, instance, args, kwargs): # noqa: | |
| if hasattr(iterator, "__await__"): | ||
| try: | ||
| iterator = await iterator | ||
| except Exception as error: | ||
| except (Exception, asyncio.CancelledError) as error: | ||
| _log_error_and_end_span(span, error) | ||
| raise | ||
|
|
||
|
|
@@ -824,6 +825,9 @@ def __init__(self, iterator: Any, span: Any, start_time: float): | |
| self._chunks: list[Any] = [] | ||
| self._finished = False | ||
|
|
||
| def __getattr__(self, name: str) -> Any: | ||
| return getattr(self._iterator, name) | ||
|
|
||
| def _record(self, event: Any) -> None: | ||
| if self._first_token_time is None: | ||
| self._first_token_time = time.time() | ||
|
|
@@ -867,7 +871,11 @@ def __exit__(self, exc_type, exc_value, traceback): | |
| context_manager = getattr(self, "_context_manager", self._iterator) | ||
| exit_method = getattr(context_manager, "__exit__", None) | ||
| if exit_method is not None: | ||
| suppress = bool(exit_method(exc_type, exc_value, traceback)) | ||
| try: | ||
| suppress = bool(exit_method(exc_type, exc_value, traceback)) | ||
| except (Exception, asyncio.CancelledError) as error: | ||
| self._finish(error=error) | ||
| raise | ||
| if exc_value is not None: | ||
| self._finish(error=exc_value) | ||
| else: | ||
|
|
@@ -883,12 +891,23 @@ def __next__(self): | |
| except StopIteration: | ||
| self._finish() | ||
| raise | ||
| except Exception as error: | ||
| except (Exception, asyncio.CancelledError) as error: | ||
| self._finish(error=error) | ||
| raise | ||
| self._record(event) | ||
| return event | ||
|
|
||
| def close(self) -> None: | ||
| close_method = getattr(self._iterator, "close", None) | ||
| try: | ||
| if callable(close_method): | ||
| close_method() | ||
| except (Exception, asyncio.CancelledError) as error: | ||
| self._finish(error=error) | ||
| raise | ||
| else: | ||
| self._finish() | ||
|
|
||
|
|
||
| class _AsyncTracedChatStream(_ChatStreamTracker): | ||
| """Async counterpart of :class:`_TracedChatStream`.""" | ||
|
|
@@ -906,7 +925,11 @@ async def __aexit__(self, exc_type, exc_value, traceback): | |
| context_manager = getattr(self, "_context_manager", self._iterator) | ||
| aexit = getattr(context_manager, "__aexit__", None) | ||
| if aexit is not None: | ||
| suppress = bool(await aexit(exc_type, exc_value, traceback)) | ||
| try: | ||
| suppress = bool(await aexit(exc_type, exc_value, traceback)) | ||
| except (Exception, asyncio.CancelledError) as error: | ||
| self._finish(error=error) | ||
| raise | ||
| if exc_value is not None: | ||
| self._finish(error=exc_value) | ||
| else: | ||
|
|
@@ -922,12 +945,36 @@ async def __anext__(self): | |
| except StopAsyncIteration: | ||
| self._finish() | ||
| raise | ||
| except Exception as error: | ||
| except (Exception, asyncio.CancelledError) as error: | ||
| self._finish(error=error) | ||
| raise | ||
| self._record(event) | ||
| return event | ||
|
|
||
| async def aclose(self) -> None: | ||
| aclose_method = getattr(self._iterator, "aclose", None) | ||
| try: | ||
| if callable(aclose_method): | ||
| res = aclose_method() | ||
| if hasattr(res, "__await__"): | ||
| await res | ||
| except (Exception, asyncio.CancelledError) as error: | ||
| self._finish(error=error) | ||
| raise | ||
| else: | ||
| self._finish() | ||
|
|
||
| def close(self) -> None: | ||
| close_method = getattr(self._iterator, "close", None) | ||
| try: | ||
| if callable(close_method): | ||
| close_method() | ||
| except (Exception, asyncio.CancelledError) as error: | ||
| self._finish(error=error) | ||
| raise | ||
| else: | ||
| self._finish() | ||
|
|
||
|
|
||
| # --------------------------------------------------------------------------- | ||
| # Manual wrapping — `wrap_cohere(client)` | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
lets avoid mocks/fakes! If we need to we should record new vcr cassettes
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Agreed on avoiding hand-written fakes where a cassette can carry the behavior — the two stream-consumption tests already reuse the checked-in VCR cassette (
test_wrap_cohere_chat_stream_v1_sync), and the subscriptions test uses a real client, so there is no stand-in client on those paths.The remaining four cases cover behavior a cassette cannot express: a task cancelled mid-
chat()/chat_stream(), and a stream whoseclose()raises. VCR replays recorded HTTP responses; there is no response to record when the caller raisesCancelledError, and a failingclose()is local cleanup rather than traffic. I used a narrowpatch.objecton the Cohere base client so the wrapper's span-finalization code still runs — the instrumentation itself is not replaced.I am happy to change the approach if you have a preferred pattern, e.g. cancelling a real VCR-backed client task at a specific await point, or using a fault-injecting transport fixture. Which do you prefer?