From 88d6397209477711677e3d916973fdc3fe5e3a08 Mon Sep 17 00:00:00 2001 From: fuchengwaridae Date: Thu, 9 Jul 2026 18:57:43 -0700 Subject: [PATCH 1/3] feat(ai): capture OpenAI Agents response cost --- .../changesets/openai-agents-response-cost.md | 5 ++ posthog/ai/openai_agents/processor.py | 5 ++ .../test/ai/openai_agents/test_processor.py | 57 ++++++++++++++++++- 3 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 .sampo/changesets/openai-agents-response-cost.md diff --git a/.sampo/changesets/openai-agents-response-cost.md b/.sampo/changesets/openai-agents-response-cost.md new file mode 100644 index 00000000..b157dd6f --- /dev/null +++ b/.sampo/changesets/openai-agents-response-cost.md @@ -0,0 +1,5 @@ +--- +'pypi/posthog': minor +--- + +feat(ai): capture pre-calculated total cost from OpenAI Agents Responses API usage. diff --git a/posthog/ai/openai_agents/processor.py b/posthog/ai/openai_agents/processor.py index d9a0de6e..1a1a5764 100644 --- a/posthog/ai/openai_agents/processor.py +++ b/posthog/ai/openai_agents/processor.py @@ -684,9 +684,11 @@ def _handle_response_span( usage = getattr(response, "usage", None) if response else None input_tokens = 0 output_tokens = 0 + total_cost = None if usage: input_tokens = getattr(usage, "input_tokens", 0) or 0 output_tokens = getattr(usage, "output_tokens", 0) or 0 + total_cost = getattr(usage, "cost", None) # Try to extract model from response model = getattr(response, "model", None) if response else None @@ -703,6 +705,9 @@ def _handle_response_span( "$ai_total_tokens": input_tokens + output_tokens, } + if total_cost is not None: + properties["$ai_total_cost_usd"] = total_cost + # Extract output content from response if response: output_items = getattr(response, "output", None) diff --git a/posthog/test/ai/openai_agents/test_processor.py b/posthog/test/ai/openai_agents/test_processor.py index 99ad7b43..a3e1ab65 100644 --- a/posthog/test/ai/openai_agents/test_processor.py +++ b/posthog/test/ai/openai_agents/test_processor.py @@ -2,6 +2,7 @@ from unittest.mock import MagicMock, patch import pytest +from openai.types.responses.response_usage import ResponseUsage try: from agents.tracing.span_data import ( @@ -533,9 +534,16 @@ def test_response_span_with_output_and_total_tokens( mock_response.id = "resp_123" mock_response.model = "gpt-4o" mock_response.output = [{"type": "message", "content": "Hello!"}] - mock_response.usage = MagicMock() - mock_response.usage.input_tokens = 25 - mock_response.usage.output_tokens = 10 + mock_response.usage = ResponseUsage.model_validate( + { + "input_tokens": 25, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 10, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 35, + "cost": 0.01234, + } + ) span_data = ResponseSpanData( response=mock_response, @@ -554,6 +562,49 @@ def test_response_span_with_output_and_total_tokens( {"type": "message", "content": "Hello!"} ] assert call_kwargs["properties"]["$ai_response_id"] == "resp_123" + assert call_kwargs["properties"]["$ai_total_cost_usd"] == 0.01234 + + def test_response_span_preserves_zero_total_cost( + self, processor, mock_client, mock_span + ): + """Test ResponseSpanData preserves an explicit zero total cost.""" + mock_response = MagicMock() + mock_response.id = "resp_123" + mock_response.model = "gpt-4o" + mock_response.output = [] + mock_response.usage = MagicMock() + mock_response.usage.input_tokens = 25 + mock_response.usage.output_tokens = 10 + mock_response.usage.cost = 0 + + mock_span.span_data = ResponseSpanData(response=mock_response) + + processor.on_span_start(mock_span) + processor.on_span_end(mock_span) + + properties = mock_client.capture.call_args[1]["properties"] + assert properties["$ai_total_cost_usd"] == 0 + + def test_response_span_omits_total_cost_when_not_provided( + self, processor, mock_client, mock_span + ): + """Test ResponseSpanData omits total cost when usage does not provide it.""" + mock_response = MagicMock() + mock_response.id = "resp_123" + mock_response.model = "gpt-4o" + mock_response.output = [] + mock_response.usage = MagicMock() + mock_response.usage.input_tokens = 25 + mock_response.usage.output_tokens = 10 + mock_response.usage.cost = None + + mock_span.span_data = ResponseSpanData(response=mock_response) + + processor.on_span_start(mock_span) + processor.on_span_end(mock_span) + + properties = mock_client.capture.call_args[1]["properties"] + assert "$ai_total_cost_usd" not in properties def test_speech_span_with_pass_through_properties( self, processor, mock_client, mock_span From 39946bc0cff5b3145749440446f9e81bf763e799 Mon Sep 17 00:00:00 2001 From: fuchengwaridae Date: Thu, 9 Jul 2026 19:05:24 -0700 Subject: [PATCH 2/3] refactor(ai): align response cost code style --- posthog/ai/openai_agents/processor.py | 7 ++- .../test/ai/openai_agents/test_processor.py | 51 +++++-------------- 2 files changed, 15 insertions(+), 43 deletions(-) diff --git a/posthog/ai/openai_agents/processor.py b/posthog/ai/openai_agents/processor.py index 1a1a5764..affeec85 100644 --- a/posthog/ai/openai_agents/processor.py +++ b/posthog/ai/openai_agents/processor.py @@ -682,13 +682,12 @@ def _handle_response_span( # Try to extract usage from response usage = getattr(response, "usage", None) if response else None + total_cost_usd = getattr(usage, "cost", None) if usage else None input_tokens = 0 output_tokens = 0 - total_cost = None if usage: input_tokens = getattr(usage, "input_tokens", 0) or 0 output_tokens = getattr(usage, "output_tokens", 0) or 0 - total_cost = getattr(usage, "cost", None) # Try to extract model from response model = getattr(response, "model", None) if response else None @@ -705,8 +704,8 @@ def _handle_response_span( "$ai_total_tokens": input_tokens + output_tokens, } - if total_cost is not None: - properties["$ai_total_cost_usd"] = total_cost + if total_cost_usd is not None: + properties["$ai_total_cost_usd"] = total_cost_usd # Extract output content from response if response: diff --git a/posthog/test/ai/openai_agents/test_processor.py b/posthog/test/ai/openai_agents/test_processor.py index a3e1ab65..3ea619e6 100644 --- a/posthog/test/ai/openai_agents/test_processor.py +++ b/posthog/test/ai/openai_agents/test_processor.py @@ -2,7 +2,6 @@ from unittest.mock import MagicMock, patch import pytest -from openai.types.responses.response_usage import ResponseUsage try: from agents.tracing.span_data import ( @@ -534,16 +533,10 @@ def test_response_span_with_output_and_total_tokens( mock_response.id = "resp_123" mock_response.model = "gpt-4o" mock_response.output = [{"type": "message", "content": "Hello!"}] - mock_response.usage = ResponseUsage.model_validate( - { - "input_tokens": 25, - "input_tokens_details": {"cached_tokens": 0}, - "output_tokens": 10, - "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 35, - "cost": 0.01234, - } - ) + mock_response.usage = MagicMock() + mock_response.usage.input_tokens = 25 + mock_response.usage.output_tokens = 10 + mock_response.usage.cost = None span_data = ResponseSpanData( response=mock_response, @@ -562,33 +555,13 @@ def test_response_span_with_output_and_total_tokens( {"type": "message", "content": "Hello!"} ] assert call_kwargs["properties"]["$ai_response_id"] == "resp_123" - assert call_kwargs["properties"]["$ai_total_cost_usd"] == 0.01234 - - def test_response_span_preserves_zero_total_cost( - self, processor, mock_client, mock_span - ): - """Test ResponseSpanData preserves an explicit zero total cost.""" - mock_response = MagicMock() - mock_response.id = "resp_123" - mock_response.model = "gpt-4o" - mock_response.output = [] - mock_response.usage = MagicMock() - mock_response.usage.input_tokens = 25 - mock_response.usage.output_tokens = 10 - mock_response.usage.cost = 0 - - mock_span.span_data = ResponseSpanData(response=mock_response) - - processor.on_span_start(mock_span) - processor.on_span_end(mock_span) - - properties = mock_client.capture.call_args[1]["properties"] - assert properties["$ai_total_cost_usd"] == 0 + assert "$ai_total_cost_usd" not in call_kwargs["properties"] - def test_response_span_omits_total_cost_when_not_provided( - self, processor, mock_client, mock_span + @pytest.mark.parametrize("total_cost_usd", [0.01234, 0.0]) + def test_response_span_includes_total_cost( + self, processor, mock_client, mock_span, total_cost_usd ): - """Test ResponseSpanData omits total cost when usage does not provide it.""" + """Test ResponseSpanData includes a pre-calculated total cost.""" mock_response = MagicMock() mock_response.id = "resp_123" mock_response.model = "gpt-4o" @@ -596,15 +569,15 @@ def test_response_span_omits_total_cost_when_not_provided( mock_response.usage = MagicMock() mock_response.usage.input_tokens = 25 mock_response.usage.output_tokens = 10 - mock_response.usage.cost = None + mock_response.usage.cost = total_cost_usd mock_span.span_data = ResponseSpanData(response=mock_response) processor.on_span_start(mock_span) processor.on_span_end(mock_span) - properties = mock_client.capture.call_args[1]["properties"] - assert "$ai_total_cost_usd" not in properties + call_kwargs = mock_client.capture.call_args[1] + assert call_kwargs["properties"]["$ai_total_cost_usd"] == total_cost_usd def test_speech_span_with_pass_through_properties( self, processor, mock_client, mock_span From 955bd1ac7396129363ffc1e9a4979cc98ecdc46d Mon Sep 17 00:00:00 2001 From: fuchengwaridae Date: Thu, 9 Jul 2026 20:24:49 -0700 Subject: [PATCH 3/3] chore: generate patch changeset with Sampo --- .sampo/changesets/doughty-iceseeker-tursas.md | 5 +++++ .sampo/changesets/openai-agents-response-cost.md | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 .sampo/changesets/doughty-iceseeker-tursas.md delete mode 100644 .sampo/changesets/openai-agents-response-cost.md diff --git a/.sampo/changesets/doughty-iceseeker-tursas.md b/.sampo/changesets/doughty-iceseeker-tursas.md new file mode 100644 index 00000000..e2cb0890 --- /dev/null +++ b/.sampo/changesets/doughty-iceseeker-tursas.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: patch +--- + +Capture pre-calculated total cost from OpenAI Agents Responses API usage. diff --git a/.sampo/changesets/openai-agents-response-cost.md b/.sampo/changesets/openai-agents-response-cost.md deleted file mode 100644 index b157dd6f..00000000 --- a/.sampo/changesets/openai-agents-response-cost.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'pypi/posthog': minor ---- - -feat(ai): capture pre-calculated total cost from OpenAI Agents Responses API usage.