From 2e5a1976027e287443bd933c1eea84758091607e Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Tue, 4 Aug 2026 09:43:15 +0200 Subject: [PATCH 1/3] feat(gooddata-eval): support targeting a specific AI Hub agent GoodData has no admin-settable "default agent": when a conversation doesn't name one, the platform picks whichever agent was last used or last edited in that workspace. Orgs with multiple AI Hub agents (e.g. one scoped to visualization only) can end up silently evaluating the wrong one -- a metric_skill/alert_skill item run against a visualization-only agent never passes, regardless of question quality. ChatClient never sent an agentId at all, so there was no way to pick. - ChatClient gains an `agent_id` param; `create_conversation()` sends `{"agentId": ...}` only when set -- omitted entirely when None, so existing behavior (platform's own default-agent resolution) is unchanged unless the caller opts in. - Threaded through all 7 run_agentic_*/evaluate_agentic_* pairs, the agentic-dispatch layer (_dispatch_agentic/run_agentic_items), and the non-agentic ChatClient construction in cli/main.py. - New `gd-eval run --agent-id ID` flag (or `GD_EVAL_AGENT_ID` env var, same precedence convention as --token/GOODDATA_TOKEN). - README: new flags-table row + a "Targeting a specific AI Hub agent" section with real usage examples. - Tests: ChatClient POST-body shape (with/without agent_id), CLI arg parsing, flag/env-var/unset precedence into the constructed ChatClient, and _dispatch_agentic threading it to evaluate_agentic_*. Co-Authored-By: Claude Sonnet 5 --- packages/gooddata-eval/README.md | 35 ++++++ .../src/gooddata_eval/cli/agentic_runner.py | 11 +- .../src/gooddata_eval/cli/main.py | 14 +++ .../gooddata_eval/core/agentic/alert_skill.py | 5 +- .../core/agentic/conversation.py | 5 +- .../core/agentic/general_question.py | 5 +- .../gooddata_eval/core/agentic/guardrail.py | 5 +- .../core/agentic/metric_skill.py | 5 +- .../gooddata_eval/core/agentic/search_tool.py | 5 +- .../core/agentic/visualization.py | 5 +- .../src/gooddata_eval/core/chat/sse_client.py | 15 ++- .../src/gooddata_eval/core/config.py | 1 + .../tests/test_agentic_runner.py | 50 ++++++++ packages/gooddata-eval/tests/test_cli.py | 110 ++++++++++++++++++ .../gooddata-eval/tests/test_sse_client.py | 30 ++++- 15 files changed, 289 insertions(+), 12 deletions(-) create mode 100644 packages/gooddata-eval/tests/test_agentic_runner.py diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index dfbeafd63..c19a340d9 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -62,6 +62,40 @@ When the same model id is offered by multiple providers, use the Both provider name and provider id are accepted as the prefix. +### Targeting a specific AI Hub agent + +GoodData has no admin-settable "default agent": when a conversation doesn't +name one, the platform picks whichever agent was last used or last edited in +that workspace. If your org has several AI Hub agents configured (e.g. one +scoped to visualization only, another with every skill enabled), evaluating +without `--agent-id` can silently exercise the wrong one — a +`metric_skill`/`alert_skill` item run against a visualization-only agent will +never pass, no matter how well-formed the question is. + +```bash +export GD_EVAL_AGENT_ID='eval-all-skills' + +gd-eval run \ + --host https://your.gooddata.cloud \ + --workspace ecommerce_demo \ + --dataset ./my-dataset \ + --model gpt-5.2 \ + --runs 1 \ + --json results.json +``` + +Or pass it explicitly instead of via the env var: + +```bash +gd-eval run \ + --host https://your.gooddata.cloud \ + --workspace ecommerce_demo \ + --dataset ./my-dataset \ + --agent-id eval-all-skills \ + --model gpt-5.2 \ + --runs 1 +``` + ### All flags #### Connection @@ -72,6 +106,7 @@ Both provider name and provider id are accepted as the prefix. | `--token TOKEN` | `GOODDATA_TOKEN` | API token. Pass via flag or env var. | | `--profile NAME` | — | Profile name in `~/.gooddata/profiles.yaml` (same file as the `gdc` CLI). | | `--workspace ID` | — | **Required.** Workspace id to evaluate against. | +| `--agent-id ID` | `GD_EVAL_AGENT_ID` | AI Hub agent every conversation should target. GoodData has no admin-settable default agent — without this, each conversation falls back to whichever agent the platform's last-used/last-edited heuristic resolves, which may not have every skill under test enabled. | #### Dataset source (pick one) diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 31104660e..d9971a5b4 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -80,6 +80,7 @@ def _dispatch_agentic( langfuse: Any, run_ts: str, model_version_override: str | None, + agent_id: str | None = None, ) -> None: """Call the appropriate evaluate_agentic_* function for the item's test_kind.""" kind = item.test_kind @@ -100,6 +101,7 @@ def _dispatch_agentic( question=item.question, expected_outputs=_parse_visualization_expected(eo), k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_metric_skill": @@ -110,6 +112,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, (dict, list)) else {}, k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_alert_skill": @@ -120,6 +123,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, dict) else {}, k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_search": @@ -133,6 +137,7 @@ def _dispatch_agentic( question=item.question, expected_tool_call=expected_args, k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_general_question": @@ -143,6 +148,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, str) else str(eo), k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_guardrail": @@ -153,6 +159,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, str) else str(eo), k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_conversation": @@ -162,6 +169,7 @@ def _dispatch_agentic( token=token, workspace_id=workspace_id, fixture=ConversationFixture.model_validate(fixture_data), + agent_id=agent_id, **lf_kw, ) else: @@ -180,6 +188,7 @@ def run_agentic_items( run_ts: str, on_item_start: Any = None, on_item_done: Any = None, + agent_id: str | None = None, ) -> EvalReport: """Run agentic items through evaluate_agentic_* and return an EvalReport.""" langfuse = make_langfuse_client() if use_langfuse else None @@ -202,7 +211,7 @@ def run_agentic_items( ) t0 = time.perf_counter() try: - _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version) + _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version, agent_id) item_report.pass_at_k = True item_report.runs = k except AssertionError as exc: diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/main.py b/packages/gooddata-eval/src/gooddata_eval/cli/main.py index 0303270be..be7a2b9e1 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/main.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/main.py @@ -2,6 +2,7 @@ """`gd-eval` command-line entry point.""" import argparse +import os import sys import threading from datetime import datetime, timezone @@ -109,6 +110,16 @@ def _build_parser() -> argparse.ArgumentParser: action="store_true", help="Log scores and traces to Langfuse (requires --langfuse-dataset and LANGFUSE_* env vars).", ) + run.add_argument( + "--agent-id", + dest="agent_id", + help=( + "AI Hub agent id every conversation should target (or set GD_EVAL_AGENT_ID). " + "GoodData has no admin-settable default agent -- without this, each conversation " + "falls back to whichever agent the platform's last-used/last-edited heuristic " + "resolves, which may not have every skill under test enabled." + ), + ) models_cmd = sub.add_parser("models", help="List LLM providers and models configured in the org.") models_cmd.add_argument("--host", help="GoodData host URL.") models_cmd.add_argument("--token", help="API token (or set GOODDATA_TOKEN).") @@ -333,6 +344,7 @@ def on_langfuse_item_done( run_ts=run_ts, on_item_start=on_item_start, on_item_done=on_item_done, + agent_id=config.agent_id, ) # --- non-agentic items (single-turn, use Evaluator) --- @@ -342,6 +354,7 @@ def on_langfuse_item_done( token=config.token, workspace_id=config.workspace_id, preserve_failed=config.preserve_failed, + agent_id=config.agent_id, ), SummaryClient(host=config.host, token=config.token, workspace_id=config.workspace_id), ) @@ -433,6 +446,7 @@ def main(argv: list[str] | None = None) -> int: quiet=args.quiet, kind=args.kind, preserve_failed=args.preserve_failed, + agent_id=args.agent_id or os.environ.get("GD_EVAL_AGENT_ID"), ) return _run(config) except ( diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index 1a7d2a188..085e45098 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -342,11 +342,12 @@ def run_agentic_alert_skill( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + agent_id: str | None = None, ) -> AgenticAlertSummary: """Run the alert-skill agentic evaluation K times and return a summary.""" expected = _normalize_expected_output(expected_output) run_results: list[AlertRunResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id) sdk = GoodDataSdk.create(host, token) def _run_once(conv_id: str) -> AlertRunResult: @@ -456,6 +457,7 @@ def evaluate_agentic_alert_skill( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "alert_skill", @@ -481,6 +483,7 @@ def evaluate_agentic_alert_skill( k=k, max_iterations=max_iterations, initial_conversation_id=initial_conversation_id, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 6b79b3279..da7e0fc91 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -278,6 +278,7 @@ def run_agentic_conversation( fixture: ConversationFixture, max_clarification_turns: int = 20, initial_conversation_id: str | None = None, + agent_id: str | None = None, ) -> ConversationResult: """Run a multi-turn, multi-skill conversation evaluation (no K-runs). @@ -285,7 +286,7 @@ def run_agentic_conversation( trigger up to *max_clarification_turns* additional rounds of simulated-user replies before the agent produces the expected output. """ - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id) sdk = GoodDataSdk.create(host, token) turn_results: list[TurnResult] = [] turn_outputs: dict[str, dict] = {} @@ -397,6 +398,7 @@ def evaluate_agentic_conversation( fixture: ConversationFixture, max_clarification_turns: int = 20, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "conversation", @@ -420,6 +422,7 @@ def evaluate_agentic_conversation( fixture=fixture, max_clarification_turns=max_clarification_turns, initial_conversation_id=initial_conversation_id, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py index 653a77956..014855ea2 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py @@ -71,10 +71,11 @@ def run_agentic_general_question( expected_output: str, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + agent_id: str | None = None, ) -> AgenticGeneralQuestionSummary: """Run the general-question agentic evaluation K times and return a summary.""" run_results: list[GeneralQuestionResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id) judge = LLMJudge(_GENERAL_QUESTION_EVALUATION_STEPS, model="gpt-4o") try: @@ -147,6 +148,7 @@ def evaluate_agentic_general_question( expected_output: str, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "general_question", @@ -171,6 +173,7 @@ def evaluate_agentic_general_question( expected_output=expected_output, k=k, initial_conversation_id=initial_conversation_id, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py index cb61da24a..16afeb2e6 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py @@ -68,10 +68,11 @@ def run_agentic_guardrail( expected_output: str, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + agent_id: str | None = None, ) -> AgenticGuardrailSummary: """Run the guardrail agentic evaluation K times and return a summary.""" run_results: list[GuardrailResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id) judge = LLMJudge(_GUARDRAIL_EVALUATION_STEPS, model="gpt-4o") try: @@ -144,6 +145,7 @@ def evaluate_agentic_guardrail( expected_output: str, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "guardrail", @@ -168,6 +170,7 @@ def evaluate_agentic_guardrail( expected_output=expected_output, k=k, initial_conversation_id=initial_conversation_id, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index a2758f6a8..d25088af8 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -232,6 +232,7 @@ def run_agentic_metric_skill( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + agent_id: str | None = None, ) -> AgenticMetricSummary: """Run the metric-skill agentic evaluation K times and return a summary. @@ -240,7 +241,7 @@ def run_agentic_metric_skill( """ expected_outputs: list[dict] = expected_output if isinstance(expected_output, list) else [expected_output] run_results: list[MetricRunResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id) sdk = GoodDataSdk.create(host, token) try: @@ -294,6 +295,7 @@ def evaluate_agentic_metric_skill( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "metric_skill", @@ -319,6 +321,7 @@ def evaluate_agentic_metric_skill( k=k, max_iterations=max_iterations, initial_conversation_id=initial_conversation_id, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py index cdbce48ab..9a92eb079 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py @@ -67,11 +67,12 @@ def run_agentic_search_tool( expected_tool_call: dict, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + agent_id: str | None = None, ) -> AgenticSearchSummary: """Run the search-tool agentic evaluation K times (single-turn each).""" run_results: list[SearchResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id) try: conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() try: @@ -138,6 +139,7 @@ def evaluate_agentic_search_tool( expected_tool_call: dict, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "search", @@ -162,6 +164,7 @@ def evaluate_agentic_search_tool( expected_tool_call=expected_tool_call, k=k, initial_conversation_id=initial_conversation_id, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index 12914157a..6abf33fa4 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -203,6 +203,7 @@ def run_agentic_visualization( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + agent_id: str | None = None, ) -> AgenticRunSummary: """Run K independent conversations and return evaluation results. @@ -211,7 +212,7 @@ def run_agentic_visualization( fresh conversations. Caller-supplied conversations are not deleted; all conversations created by this function are deleted on completion. """ - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id) run_results: list[RunResult] = [] try: @@ -258,6 +259,7 @@ def evaluate_agentic_visualization( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "visualization", @@ -285,6 +287,7 @@ def evaluate_agentic_visualization( k=k, max_iterations=max_iterations, initial_conversation_id=initial_conversation_id, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index 091436b27..649587e3e 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -242,16 +242,27 @@ class ChatClient: """Single-turn AI chat client over the GoodData AI conversation endpoints.""" def __init__( - self, host: str, token: str, workspace_id: str, *, timeout: float = 300.0, preserve_failed: bool = False + self, + host: str, + token: str, + workspace_id: str, + *, + timeout: float = 300.0, + preserve_failed: bool = False, + agent_id: str | None = None, ): self._base = f"{host.rstrip('/')}/api/v1/ai/workspaces/{workspace_id}/chat/conversations" self._auth = {"Authorization": f"Bearer {token}"} self._client = httpx.Client(timeout=timeout) self._preserve_failed = preserve_failed + self._agent_id = agent_id def create_conversation(self) -> str: def _do() -> str: - resp = self._client.post(self._base, headers={**self._auth, "Content-Type": "application/json"}) + body = {"agentId": self._agent_id} if self._agent_id else {} + resp = self._client.post( + self._base, headers={**self._auth, "Content-Type": "application/json"}, json=body + ) resp.raise_for_status() body = resp.json() if "conversationId" not in body: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/config.py b/packages/gooddata-eval/src/gooddata_eval/core/config.py index 277785176..800fa8f27 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/config.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/config.py @@ -20,3 +20,4 @@ class RunConfig: quiet: bool = False kind: str = "visualization" preserve_failed: bool = False + agent_id: str | None = None diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py new file mode 100644 index 000000000..dbbc3443e --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -0,0 +1,50 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +from unittest.mock import patch + +from gooddata_eval.cli.agentic_runner import _dispatch_agentic +from gooddata_eval.core.models import DatasetItem + + +def test_dispatch_agentic_passes_agent_id_through_to_alert_skill(): + item = DatasetItem( + id="q1", + dataset_name="ds", + test_kind="agentic_alert_skill", + question="Alert me when spend exceeds 100", + expected_output={"Operator": "GREATER_THAN", "Threshold": 100}, + ) + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill") as mock_eval: + _dispatch_agentic( + item, + host="https://h", + token="tok", + workspace_id="ws1", + k=1, + langfuse=None, + run_ts="2026-01-01", + model_version_override=None, + agent_id="agent-1", + ) + assert mock_eval.call_args.kwargs["agent_id"] == "agent-1" + + +def test_dispatch_agentic_omits_agent_id_by_default(): + item = DatasetItem( + id="q1", + dataset_name="ds", + test_kind="agentic_metric_skill", + question="Create a metric for total spend", + expected_output={"maql": "SELECT {metric/spend}"}, + ) + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_metric_skill") as mock_eval: + _dispatch_agentic( + item, + host="https://h", + token="tok", + workspace_id="ws1", + k=1, + langfuse=None, + run_ts="2026-01-01", + model_version_override=None, + ) + assert mock_eval.call_args.kwargs["agent_id"] is None diff --git a/packages/gooddata-eval/tests/test_cli.py b/packages/gooddata-eval/tests/test_cli.py index 28aecb3c4..74916760f 100644 --- a/packages/gooddata-eval/tests/test_cli.py +++ b/packages/gooddata-eval/tests/test_cli.py @@ -26,6 +26,16 @@ def test_build_run_config_requires_a_source(): cli_main.parse_args(["run", "--host", "h", "--workspace", "w"]) +def test_parse_args_agent_id_flag(): + args = cli_main.parse_args(["run", "--host", "h", "--workspace", "w", "--dataset", "d", "--agent-id", "agent-1"]) + assert args.agent_id == "agent-1" + + +def test_parse_args_agent_id_defaults_to_none(): + args = cli_main.parse_args(["run", "--host", "h", "--workspace", "w", "--dataset", "d"]) + assert args.agent_id is None + + def test_cli_run_end_to_end(monkeypatch, tmp_path, fixtures_dir): # Stub connection + model activation + chat backend so no network is needed. monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok")) @@ -93,6 +103,106 @@ def _fake_run( assert orjson.loads(out.read_bytes())["runs"]["Test Provider/gpt-5.2"]["summary"]["passed"] == 1 +def _stub_run_for_agent_id_test(monkeypatch, seen_chat_client_kwargs): + monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok")) + + class _FakeController: + def __init__(self, *a, **k): ... + def get_active(self): + return ActiveLlmProvider(provider_id="prov", default_model_id="gpt-5.2") + + def resolve_and_activate(self, requested, provider=None): + return ResolvedModel( + provider_id="prov", model_id=requested or "gpt-5.2", switched=False, provider_name="Test Provider" + ) + + def restore(self, original): ... + def close(self): ... + + monkeypatch.setattr(cli_main, "WorkspaceModelController", _FakeController) + + def _fake_run(items, backend, *, runs, model, workspace_id, **kw): + return EvalReport(model=model, workspace_id=workspace_id, items=[]) + + monkeypatch.setattr(cli_main, "run_items", _fake_run) + + def _spy_chat_client(**kwargs): + seen_chat_client_kwargs.update(kwargs) + return object() + + monkeypatch.setattr(cli_main, "ChatClient", _spy_chat_client) + + +def test_cli_run_passes_agent_id_flag_to_chat_client(monkeypatch, tmp_path, fixtures_dir): + seen = {} + _stub_run_for_agent_id_test(monkeypatch, seen) + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--agent-id", + "agent-1", + "--json", + str(tmp_path / "res.json"), + ] + ) + assert exit_code == 0 + assert seen["agent_id"] == "agent-1" + + +def test_cli_run_falls_back_to_agent_id_env_var(monkeypatch, tmp_path, fixtures_dir): + monkeypatch.setenv("GD_EVAL_AGENT_ID", "agent-from-env") + seen = {} + _stub_run_for_agent_id_test(monkeypatch, seen) + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--json", + str(tmp_path / "res.json"), + ] + ) + assert exit_code == 0 + assert seen["agent_id"] == "agent-from-env" + + +def test_cli_run_agent_id_omitted_when_unset(monkeypatch, tmp_path, fixtures_dir): + monkeypatch.delenv("GD_EVAL_AGENT_ID", raising=False) + seen = {} + _stub_run_for_agent_id_test(monkeypatch, seen) + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--json", + str(tmp_path / "res.json"), + ] + ) + assert exit_code == 0 + assert seen["agent_id"] is None + + def test_cli_operational_error_exits_nonzero(monkeypatch, fixtures_dir): def _boom(host, token, profile): raise ConnectionError_("Missing token.") diff --git a/packages/gooddata-eval/tests/test_sse_client.py b/packages/gooddata-eval/tests/test_sse_client.py index c5590e428..8da0154e4 100644 --- a/packages/gooddata-eval/tests/test_sse_client.py +++ b/packages/gooddata-eval/tests/test_sse_client.py @@ -145,8 +145,8 @@ def test_parse_sse_lines_non_retryable_status_is_chat_error_not_transient(): assert ei.value.status_code == 400 -def _client_with_handler(handler): - client = ChatClient(host="https://example.invalid", token="t", workspace_id="w") +def _client_with_handler(handler, **client_kwargs): + client = ChatClient(host="https://example.invalid", token="t", workspace_id="w", **client_kwargs) client._client = httpx.Client(transport=httpx.MockTransport(handler)) return client @@ -238,6 +238,32 @@ def handler(request): assert sleeps == [] +def test_create_conversation_omits_agent_id_by_default(): + # No agent_id given -> unchanged, existing behavior: GoodData's own + # last-used/last-edited default-agent resolution still applies. + seen = {} + + def handler(request): + seen["body"] = request.content + return httpx.Response(200, json={"conversationId": "abc"}) + + client = _client_with_handler(handler) + client.create_conversation() + assert seen["body"] in (b"", b"{}") + + +def test_create_conversation_sends_agent_id_when_given(): + seen = {} + + def handler(request): + seen["body"] = json.loads(request.content) + return httpx.Response(200, json={"conversationId": "abc"}) + + client = _client_with_handler(handler, agent_id="agent-123") + client.create_conversation() + assert seen["body"] == {"agentId": "agent-123"} + + def test_int_env_uses_default_when_unset(monkeypatch): monkeypatch.delenv("GD_TEST_INT", raising=False) assert sse_mod._int_env("GD_TEST_INT", 5) == 5 From 0cd64338dd25a3d315903e1090449bad25955137 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Tue, 4 Aug 2026 09:48:36 +0200 Subject: [PATCH 2/3] fix: satisfy ruff format/lint on CI - sse_client.py: reformat create_conversation() to ruff's line-length rule (the PR's own change). - test_cli.py: drop an unused `original_chat_client` local -- pre-existing on master (same line, unrelated to this PR), but ruff check runs whole-file and blocks this PR's lint-and-format-check job since this test function lives in a file the PR also touches. Co-Authored-By: Claude Sonnet 5 --- .../gooddata-eval/src/gooddata_eval/core/chat/sse_client.py | 4 +--- packages/gooddata-eval/tests/test_cli.py | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index 649587e3e..df54ddda6 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -260,9 +260,7 @@ def __init__( def create_conversation(self) -> str: def _do() -> str: body = {"agentId": self._agent_id} if self._agent_id else {} - resp = self._client.post( - self._base, headers={**self._auth, "Content-Type": "application/json"}, json=body - ) + resp = self._client.post(self._base, headers={**self._auth, "Content-Type": "application/json"}, json=body) resp.raise_for_status() body = resp.json() if "conversationId" not in body: diff --git a/packages/gooddata-eval/tests/test_cli.py b/packages/gooddata-eval/tests/test_cli.py index 74916760f..0f7735956 100644 --- a/packages/gooddata-eval/tests/test_cli.py +++ b/packages/gooddata-eval/tests/test_cli.py @@ -645,8 +645,6 @@ def close(self): ... monkeypatch.setattr(cli_main, "WorkspaceModelController", _FakeController) - original_chat_client = cli_main.ChatClient - def _capture_chat_client(**kwargs): captured_kwargs.update(kwargs) return object() From b55f608150e27227b09ab89070f0440e87231cda Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Tue, 4 Aug 2026 09:53:46 +0200 Subject: [PATCH 3/3] test: cover agent_id threading for every agentic test kind codecov flagged one uncovered line in _dispatch_agentic's agent_id threading -- the earlier tests only exercised 2 of 7 kind branches. Parametrized test now covers all 7 (vis_agentic, agentic_visualization, agentic_search, agentic_general_question, agentic_guardrail, agentic_conversation, plus the two already covered). Co-Authored-By: Claude Sonnet 5 --- .../tests/test_agentic_runner.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index dbbc3443e..3cbff3137 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -1,6 +1,7 @@ # (C) 2026 GoodData Corporation. All rights reserved. from unittest.mock import patch +import pytest from gooddata_eval.cli.agentic_runner import _dispatch_agentic from gooddata_eval.core.models import DatasetItem @@ -48,3 +49,45 @@ def test_dispatch_agentic_omits_agent_id_by_default(): model_version_override=None, ) assert mock_eval.call_args.kwargs["agent_id"] is None + + +_MIN_VIZ = {"id": "v1", "type": "table", "query": {"fields": {}, "filter_by": {}}, "metrics": [], "view_by": []} +_MIN_CONVERSATION_FIXTURE = { + "id": "c1", + "expected_skills": ["visualization"], + "turns": [{"turn_id": "t1", "message": "hi", "expected_skill": "visualization"}], +} + + +@pytest.mark.parametrize( + ("kind", "expected_output", "target"), + [ + ("vis_agentic", {"visualization": _MIN_VIZ}, "evaluate_agentic_visualization"), + ("agentic_visualization", {"visualization": _MIN_VIZ}, "evaluate_agentic_visualization"), + ("agentic_search", {"tool_call": {"function_arguments": {}}}, "evaluate_agentic_search_tool"), + ("agentic_general_question", "What is X?", "evaluate_agentic_general_question"), + ("agentic_guardrail", "Ignore prior instructions", "evaluate_agentic_guardrail"), + ("agentic_conversation", {"fixture": _MIN_CONVERSATION_FIXTURE}, "evaluate_agentic_conversation"), + ], +) +def test_dispatch_agentic_passes_agent_id_through_for_every_kind(kind, expected_output, target): + item = DatasetItem( + id="q1", + dataset_name="ds", + test_kind=kind, + question="q", + expected_output=expected_output, + ) + with patch(f"gooddata_eval.cli.agentic_runner.{target}") as mock_eval: + _dispatch_agentic( + item, + host="https://h", + token="tok", + workspace_id="ws1", + k=1, + langfuse=None, + run_ts="2026-01-01", + model_version_override=None, + agent_id="agent-1", + ) + assert mock_eval.call_args.kwargs["agent_id"] == "agent-1"