feat(gooddata-eval): support targeting a specific AI Hub agent

packages/gooddata-eval/README.md

@@ -62,6 +62,40 @@ When the same model id is offered by multiple providers, use the
 
 Both provider name and provider id are accepted as the prefix.
 
+### Targeting a specific AI Hub agent
+
+GoodData has no admin-settable "default agent": when a conversation doesn't
+name one, the platform picks whichever agent was last used or last edited in
+that workspace. If your org has several AI Hub agents configured (e.g. one
+scoped to visualization only, another with every skill enabled), evaluating
+without `--agent-id` can silently exercise the wrong one — a
+`metric_skill`/`alert_skill` item run against a visualization-only agent will
+never pass, no matter how well-formed the question is.
+
+```bash
+export GD_EVAL_AGENT_ID='eval-all-skills'
+
+gd-eval run \
+  --host  https://your.gooddata.cloud \
+  --workspace  ecommerce_demo \
+  --dataset  ./my-dataset \
+  --model  gpt-5.2 \
+  --runs  1 \
+  --json  results.json
+```
+
+Or pass it explicitly instead of via the env var:
+
+```bash
+gd-eval run \
+  --host  https://your.gooddata.cloud \
+  --workspace  ecommerce_demo \
+  --dataset  ./my-dataset \
+  --agent-id  eval-all-skills \
+  --model  gpt-5.2 \
+  --runs  1
+```
+
 ### All flags
 
 #### Connection
@@ -72,6 +106,7 @@ Both provider name and provider id are accepted as the prefix.
 | `--token TOKEN` | `GOODDATA_TOKEN` | API token. Pass via flag or env var. |
 | `--profile NAME` | — | Profile name in `~/.gooddata/profiles.yaml` (same file as the `gdc` CLI). |
 | `--workspace ID` | — | **Required.** Workspace id to evaluate against. |
+| `--agent-id ID` | `GD_EVAL_AGENT_ID` | AI Hub agent every conversation should target. GoodData has no admin-settable default agent — without this, each conversation falls back to whichever agent the platform's last-used/last-edited heuristic resolves, which may not have every skill under test enabled. |
 
 #### Dataset source (pick one)
 

packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py

@@ -80,6 +80,7 @@ def _dispatch_agentic(
     langfuse: Any,
     run_ts: str,
     model_version_override: str | None,
+    agent_id: str | None = None,
 ) -> None:
     """Call the appropriate evaluate_agentic_* function for the item's test_kind."""
     kind = item.test_kind
@@ -100,6 +101,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_outputs=_parse_visualization_expected(eo),
             k=k,
+            agent_id=agent_id,
             **lf_kw,
         )
     elif kind == "agentic_metric_skill":
@@ -110,6 +112,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_output=eo if isinstance(eo, (dict, list)) else {},
             k=k,
+            agent_id=agent_id,
             **lf_kw,
         )
     elif kind == "agentic_alert_skill":
@@ -120,6 +123,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_output=eo if isinstance(eo, dict) else {},
             k=k,
+            agent_id=agent_id,
             **lf_kw,
         )
     elif kind == "agentic_search":
@@ -133,6 +137,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_tool_call=expected_args,
             k=k,
+            agent_id=agent_id,
             **lf_kw,
         )
     elif kind == "agentic_general_question":
@@ -143,6 +148,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_output=eo if isinstance(eo, str) else str(eo),
             k=k,
+            agent_id=agent_id,
             **lf_kw,
         )
     elif kind == "agentic_guardrail":
@@ -153,6 +159,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_output=eo if isinstance(eo, str) else str(eo),
             k=k,
+            agent_id=agent_id,
             **lf_kw,
         )
     elif kind == "agentic_conversation":
@@ -162,6 +169,7 @@ def _dispatch_agentic(
             token=token,
             workspace_id=workspace_id,
             fixture=ConversationFixture.model_validate(fixture_data),
+            agent_id=agent_id,
             **lf_kw,
         )
     else:
@@ -180,6 +188,7 @@ def run_agentic_items(
     run_ts: str,
     on_item_start: Any = None,
     on_item_done: Any = None,
+    agent_id: str | None = None,
 ) -> EvalReport:
     """Run agentic items through evaluate_agentic_* and return an EvalReport."""
     langfuse = make_langfuse_client() if use_langfuse else None
@@ -202,7 +211,7 @@ def run_agentic_items(
         )
         t0 = time.perf_counter()
         try:
-            _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version)
+            _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version, agent_id)
             item_report.pass_at_k = True
             item_report.runs = k
         except AssertionError as exc:

packages/gooddata-eval/src/gooddata_eval/cli/main.py

@@ -2,6 +2,7 @@
 """`gd-eval` command-line entry point."""
 
 import argparse
+import os
 import sys
 import threading
 from datetime import datetime, timezone
@@ -109,6 +110,16 @@ def _build_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="Log scores and traces to Langfuse (requires --langfuse-dataset and LANGFUSE_* env vars).",
     )
+    run.add_argument(
+        "--agent-id",
+        dest="agent_id",
+        help=(
+            "AI Hub agent id every conversation should target (or set GD_EVAL_AGENT_ID). "
+            "GoodData has no admin-settable default agent -- without this, each conversation "
+            "falls back to whichever agent the platform's last-used/last-edited heuristic "
+            "resolves, which may not have every skill under test enabled."
+        ),
+    )
     models_cmd = sub.add_parser("models", help="List LLM providers and models configured in the org.")
     models_cmd.add_argument("--host", help="GoodData host URL.")
     models_cmd.add_argument("--token", help="API token (or set GOODDATA_TOKEN).")
@@ -333,6 +344,7 @@ def on_langfuse_item_done(
                     run_ts=run_ts,
                     on_item_start=on_item_start,
                     on_item_done=on_item_done,
+                    agent_id=config.agent_id,
                 )
 
             # --- non-agentic items (single-turn, use Evaluator) ---
@@ -342,6 +354,7 @@ def on_langfuse_item_done(
                     token=config.token,
                     workspace_id=config.workspace_id,
                     preserve_failed=config.preserve_failed,
+                    agent_id=config.agent_id,
                 ),
                 SummaryClient(host=config.host, token=config.token, workspace_id=config.workspace_id),
             )
@@ -433,6 +446,7 @@ def main(argv: list[str] | None = None) -> int:
             quiet=args.quiet,
             kind=args.kind,
             preserve_failed=args.preserve_failed,
+            agent_id=args.agent_id or os.environ.get("GD_EVAL_AGENT_ID"),
         )
         return _run(config)
     except (

packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py

@@ -342,11 +342,12 @@ def run_agentic_alert_skill(
     k: int = _DEFAULT_K,
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
+    agent_id: str | None = None,
 ) -> AgenticAlertSummary:
     """Run the alert-skill agentic evaluation K times and return a summary."""
     expected = _normalize_expected_output(expected_output)
     run_results: list[AlertRunResult] = []
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id)
     sdk = GoodDataSdk.create(host, token)
 
     def _run_once(conv_id: str) -> AlertRunResult:
@@ -456,6 +457,7 @@ def evaluate_agentic_alert_skill(
     k: int = _DEFAULT_K,
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
+    agent_id: str | None = None,
     langfuse: object | None = None,
     dataset_item_id: str = "",
     dataset_name: str = "alert_skill",
@@ -481,6 +483,7 @@ def evaluate_agentic_alert_skill(
         k=k,
         max_iterations=max_iterations,
         initial_conversation_id=initial_conversation_id,
+        agent_id=agent_id,
     )
 
     if langfuse is not None and dataset_item_id:

packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py

@@ -278,14 +278,15 @@ def run_agentic_conversation(
     fixture: ConversationFixture,
     max_clarification_turns: int = 20,
     initial_conversation_id: str | None = None,
+    agent_id: str | None = None,
 ) -> ConversationResult:
     """Run a multi-turn, multi-skill conversation evaluation (no K-runs).
 
     A single conversation is used for all turns in the fixture.  Each turn may
     trigger up to *max_clarification_turns* additional rounds of simulated-user
     replies before the agent produces the expected output.
     """
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id)
     sdk = GoodDataSdk.create(host, token)
     turn_results: list[TurnResult] = []
     turn_outputs: dict[str, dict] = {}
@@ -397,6 +398,7 @@ def evaluate_agentic_conversation(
     fixture: ConversationFixture,
     max_clarification_turns: int = 20,
     initial_conversation_id: str | None = None,
+    agent_id: str | None = None,
     langfuse: object | None = None,
     dataset_item_id: str = "",
     dataset_name: str = "conversation",
@@ -420,6 +422,7 @@ def evaluate_agentic_conversation(
         fixture=fixture,
         max_clarification_turns=max_clarification_turns,
         initial_conversation_id=initial_conversation_id,
+        agent_id=agent_id,
     )
 
     if langfuse is not None and dataset_item_id:

packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py

@@ -71,10 +71,11 @@ def run_agentic_general_question(
     expected_output: str,
     k: int = _DEFAULT_K,
     initial_conversation_id: str | None = None,
+    agent_id: str | None = None,
 ) -> AgenticGeneralQuestionSummary:
     """Run the general-question agentic evaluation K times and return a summary."""
     run_results: list[GeneralQuestionResult] = []
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id)
     judge = LLMJudge(_GENERAL_QUESTION_EVALUATION_STEPS, model="gpt-4o")
 
     try:
@@ -147,6 +148,7 @@ def evaluate_agentic_general_question(
     expected_output: str,
     k: int = _DEFAULT_K,
     initial_conversation_id: str | None = None,
+    agent_id: str | None = None,
     langfuse: object | None = None,
     dataset_item_id: str = "",
     dataset_name: str = "general_question",
@@ -171,6 +173,7 @@ def evaluate_agentic_general_question(
         expected_output=expected_output,
         k=k,
         initial_conversation_id=initial_conversation_id,
+        agent_id=agent_id,
     )
 
     if langfuse is not None and dataset_item_id:

packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py

@@ -68,10 +68,11 @@ def run_agentic_guardrail(
     expected_output: str,
     k: int = _DEFAULT_K,
     initial_conversation_id: str | None = None,
+    agent_id: str | None = None,
 ) -> AgenticGuardrailSummary:
     """Run the guardrail agentic evaluation K times and return a summary."""
     run_results: list[GuardrailResult] = []
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id)
     judge = LLMJudge(_GUARDRAIL_EVALUATION_STEPS, model="gpt-4o")
 
     try:
@@ -144,6 +145,7 @@ def evaluate_agentic_guardrail(
     expected_output: str,
     k: int = _DEFAULT_K,
     initial_conversation_id: str | None = None,
+    agent_id: str | None = None,
     langfuse: object | None = None,
     dataset_item_id: str = "",
     dataset_name: str = "guardrail",
@@ -168,6 +170,7 @@ def evaluate_agentic_guardrail(
         expected_output=expected_output,
         k=k,
         initial_conversation_id=initial_conversation_id,
+        agent_id=agent_id,
     )
 
     if langfuse is not None and dataset_item_id:

packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py

@@ -232,6 +232,7 @@ def run_agentic_metric_skill(
     k: int = _DEFAULT_K,
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
+    agent_id: str | None = None,
 ) -> AgenticMetricSummary:
     """Run the metric-skill agentic evaluation K times and return a summary.
 
@@ -240,7 +241,7 @@ def run_agentic_metric_skill(
     """
     expected_outputs: list[dict] = expected_output if isinstance(expected_output, list) else [expected_output]
     run_results: list[MetricRunResult] = []
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id)
     sdk = GoodDataSdk.create(host, token)
 
     try:
@@ -294,6 +295,7 @@ def evaluate_agentic_metric_skill(
     k: int = _DEFAULT_K,
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
+    agent_id: str | None = None,
     langfuse: object | None = None,
     dataset_item_id: str = "",
     dataset_name: str = "metric_skill",
@@ -319,6 +321,7 @@ def evaluate_agentic_metric_skill(
         k=k,
         max_iterations=max_iterations,
         initial_conversation_id=initial_conversation_id,
+        agent_id=agent_id,
     )
 
     if langfuse is not None and dataset_item_id:

packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py

@@ -67,11 +67,12 @@ def run_agentic_search_tool(
     expected_tool_call: dict,
     k: int = _DEFAULT_K,
     initial_conversation_id: str | None = None,
+    agent_id: str | None = None,
 ) -> AgenticSearchSummary:
     """Run the search-tool agentic evaluation K times (single-turn each)."""
     run_results: list[SearchResult] = []
 
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id)
     try:
         conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation()
         try:
@@ -138,6 +139,7 @@ def evaluate_agentic_search_tool(
     expected_tool_call: dict,
     k: int = _DEFAULT_K,
     initial_conversation_id: str | None = None,
+    agent_id: str | None = None,
     langfuse: object | None = None,
     dataset_item_id: str = "",
     dataset_name: str = "search",
@@ -162,6 +164,7 @@ def evaluate_agentic_search_tool(
         expected_tool_call=expected_tool_call,
         k=k,
         initial_conversation_id=initial_conversation_id,
+        agent_id=agent_id,
     )
 
     if langfuse is not None and dataset_item_id:

packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py

@@ -203,6 +203,7 @@ def run_agentic_visualization(
     k: int = _DEFAULT_K,
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
+    agent_id: str | None = None,
 ) -> AgenticRunSummary:
     """Run K independent conversations and return evaluation results.
 
@@ -211,7 +212,7 @@ def run_agentic_visualization(
     fresh conversations. Caller-supplied conversations are not deleted; all
     conversations created by this function are deleted on completion.
     """
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id)
     run_results: list[RunResult] = []
 
     try:
@@ -258,6 +259,7 @@ def evaluate_agentic_visualization(
     k: int = _DEFAULT_K,
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
+    agent_id: str | None = None,
     langfuse: object | None = None,
     dataset_item_id: str = "",
     dataset_name: str = "visualization",
@@ -285,6 +287,7 @@ def evaluate_agentic_visualization(
         k=k,
         max_iterations=max_iterations,
         initial_conversation_id=initial_conversation_id,
+        agent_id=agent_id,
     )
 
     if langfuse is not None and dataset_item_id: