From e26263c7abd58a9796528bb23134e2c778478486 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Tue, 4 Aug 2026 10:27:23 +0200 Subject: [PATCH 1/2] feat(gooddata-eval): support requesting a per-message reasoning effort GoodData Cloud's chat-conversations endpoint accepts an experimental options.reasoningEffort (LOW/MEDIUM/HIGH) on each POST .../messages call. Thread it through ChatClient.send_message/ask, all 7 agentic evaluators, the CLI dispatcher, and RunConfig, exposed as --reasoning-effort / GD_EVAL_REASONING_EFFORT. Not persisted server-side, so every message the client sends must carry it (unlike agentId, which is set once at conversation creation). --- packages/gooddata-eval/README.md | 20 +++ .../src/gooddata_eval/cli/agentic_runner.py | 11 +- .../src/gooddata_eval/cli/main.py | 20 ++- .../gooddata_eval/core/agentic/alert_skill.py | 5 +- .../core/agentic/conversation.py | 5 +- .../core/agentic/general_question.py | 7 +- .../gooddata_eval/core/agentic/guardrail.py | 7 +- .../core/agentic/metric_skill.py | 10 +- .../gooddata_eval/core/agentic/search_tool.py | 7 +- .../core/agentic/visualization.py | 16 +- .../src/gooddata_eval/core/chat/sse_client.py | 10 +- .../src/gooddata_eval/core/config.py | 1 + .../tests/test_agentic_runner.py | 67 ++++++++ .../tests/test_agentic_visualization.py | 6 +- packages/gooddata-eval/tests/test_cli.py | 157 +++++++++++++++++- .../gooddata-eval/tests/test_sse_client.py | 24 +++ 16 files changed, 346 insertions(+), 27 deletions(-) create mode 100644 packages/gooddata-eval/tests/test_agentic_runner.py diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index dfbeafd63..fe63f0d38 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -92,6 +92,7 @@ Both provider name and provider id are accepted as the prefix. |---|---|---| | `--runs K` | `2` | Independent runs per item (pass@K). An item passes if any run passes. | | `--concurrency K` | `1` | Number of items evaluated concurrently. `1` = sequential (default). Increase to load-test the agent under simultaneous requests. Progress output interleaves when K > 1. | +| `--reasoning-effort {LOW,MEDIUM,HIGH}` | — | Requested LLM reasoning effort, sent with every message this run makes (or set `GD_EVAL_REASONING_EFFORT`). See [Requesting a reasoning effort](#requesting-a-reasoning-effort) below. | #### Output @@ -106,6 +107,25 @@ Both provider name and provider id are accepted as the prefix. |---|---| | `--langfuse` | Log scores and traces to Langfuse after each item. Requires `--langfuse-dataset`. Creates one named experiment run per model (`gd-eval-{timestamp}-{model}`). Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST`. | +### Requesting a reasoning effort + +GoodData Cloud has an experimental per-message `reasoningEffort` option (`LOW`, `MEDIUM`, or `HIGH`) that hints how much the LLM should reason before answering. `gd-eval` can send it on every message it makes: + +```bash +# One-off, via flag +gd-eval run --workspace my-ws --dataset ./data --reasoning-effort HIGH + +# Session-wide, via env var +export GD_EVAL_REASONING_EFFORT=LOW +gd-eval run --workspace my-ws --dataset ./data +``` + +Things to know before using it: + +- **Not persisted server-side.** The setting applies only to the messages this run sends — it is not saved as a conversation or workspace default. Every `gd-eval` message in the run carries the value; nothing else on the server is affected. +- **Feature-flag gated.** GoodData Cloud must have the reasoning-effort feature enabled for the org; when it isn't, the value is ignored and the platform falls back to `MEDIUM` regardless of what was requested. +- **A hint, not a hard budget.** Providers with adaptive-thinking models (e.g. Anthropic, Bedrock) treat the value as a hint rather than an exact token allocation, so actual reasoning depth can still vary by model. + ### JSON report shape The JSON report always uses the nested multi-model shape: diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 31104660e..f630d5e9d 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -80,6 +80,7 @@ def _dispatch_agentic( langfuse: Any, run_ts: str, model_version_override: str | None, + reasoning_effort: str | None = None, ) -> None: """Call the appropriate evaluate_agentic_* function for the item's test_kind.""" kind = item.test_kind @@ -100,6 +101,7 @@ def _dispatch_agentic( question=item.question, expected_outputs=_parse_visualization_expected(eo), k=k, + reasoning_effort=reasoning_effort, **lf_kw, ) elif kind == "agentic_metric_skill": @@ -110,6 +112,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, (dict, list)) else {}, k=k, + reasoning_effort=reasoning_effort, **lf_kw, ) elif kind == "agentic_alert_skill": @@ -120,6 +123,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, dict) else {}, k=k, + reasoning_effort=reasoning_effort, **lf_kw, ) elif kind == "agentic_search": @@ -133,6 +137,7 @@ def _dispatch_agentic( question=item.question, expected_tool_call=expected_args, k=k, + reasoning_effort=reasoning_effort, **lf_kw, ) elif kind == "agentic_general_question": @@ -143,6 +148,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, str) else str(eo), k=k, + reasoning_effort=reasoning_effort, **lf_kw, ) elif kind == "agentic_guardrail": @@ -153,6 +159,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, str) else str(eo), k=k, + reasoning_effort=reasoning_effort, **lf_kw, ) elif kind == "agentic_conversation": @@ -162,6 +169,7 @@ def _dispatch_agentic( token=token, workspace_id=workspace_id, fixture=ConversationFixture.model_validate(fixture_data), + reasoning_effort=reasoning_effort, **lf_kw, ) else: @@ -180,6 +188,7 @@ def run_agentic_items( run_ts: str, on_item_start: Any = None, on_item_done: Any = None, + reasoning_effort: str | None = None, ) -> EvalReport: """Run agentic items through evaluate_agentic_* and return an EvalReport.""" langfuse = make_langfuse_client() if use_langfuse else None @@ -202,7 +211,7 @@ def run_agentic_items( ) t0 = time.perf_counter() try: - _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version) + _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort) item_report.pass_at_k = True item_report.runs = k except AssertionError as exc: diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/main.py b/packages/gooddata-eval/src/gooddata_eval/cli/main.py index 0303270be..8f08d169f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/main.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/main.py @@ -2,6 +2,7 @@ """`gd-eval` command-line entry point.""" import argparse +import os import sys import threading from datetime import datetime, timezone @@ -37,14 +38,15 @@ class _RoutingBackend: else uses the conversational chat endpoint. """ - def __init__(self, chat: ChatClient, summary: SummaryClient): + def __init__(self, chat: ChatClient, summary: SummaryClient, *, reasoning_effort: str | None = None): self._chat = chat self._summary = summary + self._reasoning_effort = reasoning_effort def ask(self, item: DatasetItem) -> ChatResult: if item.test_kind == _SUMMARY_TEST_KIND: return self._summary.ask(item) - return self._chat.ask(item) + return self._chat.ask(item, reasoning_effort=self._reasoning_effort) def close(self) -> None: for backend in (self._chat, self._summary): @@ -109,6 +111,17 @@ def _build_parser() -> argparse.ArgumentParser: action="store_true", help="Log scores and traces to Langfuse (requires --langfuse-dataset and LANGFUSE_* env vars).", ) + run.add_argument( + "--reasoning-effort", + dest="reasoning_effort", + choices=["LOW", "MEDIUM", "HIGH"], + default=None, + help=( + "Requested LLM reasoning effort for this run's messages (or set GD_EVAL_REASONING_EFFORT). " + "Experimental GoodData feature, gated behind an org-level flag — when disabled, the value is " + "ignored and MEDIUM is used. Not persisted server-side: applies only to messages this run sends." + ), + ) models_cmd = sub.add_parser("models", help="List LLM providers and models configured in the org.") models_cmd.add_argument("--host", help="GoodData host URL.") models_cmd.add_argument("--token", help="API token (or set GOODDATA_TOKEN).") @@ -333,6 +346,7 @@ def on_langfuse_item_done( run_ts=run_ts, on_item_start=on_item_start, on_item_done=on_item_done, + reasoning_effort=config.reasoning_effort, ) # --- non-agentic items (single-turn, use Evaluator) --- @@ -344,6 +358,7 @@ def on_langfuse_item_done( preserve_failed=config.preserve_failed, ), SummaryClient(host=config.host, token=config.token, workspace_id=config.workspace_id), + reasoning_effort=config.reasoning_effort, ) try: single_report = run_items( @@ -433,6 +448,7 @@ def main(argv: list[str] | None = None) -> int: quiet=args.quiet, kind=args.kind, preserve_failed=args.preserve_failed, + reasoning_effort=args.reasoning_effort or os.environ.get("GD_EVAL_REASONING_EFFORT"), ) return _run(config) except ( diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index 1a7d2a188..10f421793 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -342,6 +342,7 @@ def run_agentic_alert_skill( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + reasoning_effort: str | None = None, ) -> AgenticAlertSummary: """Run the alert-skill agentic evaluation K times and return a summary.""" expected = _normalize_expected_output(expected_output) @@ -361,7 +362,7 @@ def _run_once(conv_id: str) -> AlertRunResult: current_question = question for _iteration in range(max_iterations): - chat_result = client.send_message(conv_id, current_question) + chat_result = client.send_message(conv_id, current_question, reasoning_effort=reasoning_effort) alert_id, actual_args, tool_called = _extract_alert_call(chat_result.tool_call_events or []) if tool_called: alert_id_to_delete = alert_id @@ -462,6 +463,7 @@ def evaluate_agentic_alert_skill( run_timestamp: str | None = None, model_version_override: str | None = None, run_metadata_extra: dict | None = None, + reasoning_effort: str | None = None, ) -> None: """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -481,6 +483,7 @@ def evaluate_agentic_alert_skill( k=k, max_iterations=max_iterations, initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 6b79b3279..2e728472b 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -278,6 +278,7 @@ def run_agentic_conversation( fixture: ConversationFixture, max_clarification_turns: int = 20, initial_conversation_id: str | None = None, + reasoning_effort: str | None = None, ) -> ConversationResult: """Run a multi-turn, multi-skill conversation evaluation (no K-runs). @@ -315,7 +316,7 @@ def run_agentic_conversation( final_result: ChatResult | None = None for _iter in range(max_clarification_turns + 1): - chat_result = client.send_message(conversation_id, current_message) + chat_result = client.send_message(conversation_id, current_message, reasoning_effort=reasoning_effort) final_result = chat_result all_tool_calls.extend(chat_result.tool_call_events or []) @@ -403,6 +404,7 @@ def evaluate_agentic_conversation( run_timestamp: str | None = None, model_version_override: str | None = None, run_metadata_extra: dict | None = None, + reasoning_effort: str | None = None, ) -> None: """Run conversation evaluation, log to Langfuse, and raise on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -420,6 +422,7 @@ def evaluate_agentic_conversation( fixture=fixture, max_clarification_turns=max_clarification_turns, initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py index 653a77956..4c53f0fd1 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py @@ -71,6 +71,7 @@ def run_agentic_general_question( expected_output: str, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + reasoning_effort: str | None = None, ) -> AgenticGeneralQuestionSummary: """Run the general-question agentic evaluation K times and return a summary.""" run_results: list[GeneralQuestionResult] = [] @@ -80,7 +81,7 @@ def run_agentic_general_question( try: conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() try: - chat_result = client.send_message(conv_id_0, question) + chat_result = client.send_message(conv_id_0, question, reasoning_effort=reasoning_effort) actual_output = (chat_result.text_response or "").strip() passed, reasoning = judge.score( input=question, expected_output=expected_output, actual_output=actual_output @@ -102,7 +103,7 @@ def run_agentic_general_question( for _ in range(1, k): conv_id = client.create_conversation() try: - chat_result = client.send_message(conv_id, question) + chat_result = client.send_message(conv_id, question, reasoning_effort=reasoning_effort) actual_output = (chat_result.text_response or "").strip() passed, reasoning = judge.score( input=question, expected_output=expected_output, actual_output=actual_output @@ -153,6 +154,7 @@ def evaluate_agentic_general_question( run_timestamp: str | None = None, model_version_override: str | None = None, run_metadata_extra: dict | None = None, + reasoning_effort: str | None = None, ) -> None: """Run general-question evaluation, log to Langfuse, and raise on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -171,6 +173,7 @@ def evaluate_agentic_general_question( expected_output=expected_output, k=k, initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py index cb61da24a..ed694be24 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py @@ -68,6 +68,7 @@ def run_agentic_guardrail( expected_output: str, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + reasoning_effort: str | None = None, ) -> AgenticGuardrailSummary: """Run the guardrail agentic evaluation K times and return a summary.""" run_results: list[GuardrailResult] = [] @@ -77,7 +78,7 @@ def run_agentic_guardrail( try: conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() try: - chat_result = client.send_message(conv_id_0, question) + chat_result = client.send_message(conv_id_0, question, reasoning_effort=reasoning_effort) actual_output = (chat_result.text_response or "").strip() passed, reasoning = judge.score( input=question, expected_output=expected_output, actual_output=actual_output @@ -99,7 +100,7 @@ def run_agentic_guardrail( for _ in range(1, k): conv_id = client.create_conversation() try: - chat_result = client.send_message(conv_id, question) + chat_result = client.send_message(conv_id, question, reasoning_effort=reasoning_effort) actual_output = (chat_result.text_response or "").strip() passed, reasoning = judge.score( input=question, expected_output=expected_output, actual_output=actual_output @@ -150,6 +151,7 @@ def evaluate_agentic_guardrail( run_timestamp: str | None = None, model_version_override: str | None = None, run_metadata_extra: dict | None = None, + reasoning_effort: str | None = None, ) -> None: """Run guardrail evaluation, log to Langfuse, and raise on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -168,6 +170,7 @@ def evaluate_agentic_guardrail( expected_output=expected_output, k=k, initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index a2758f6a8..8224a0df6 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -179,6 +179,7 @@ def _execute_single_metric_run( question: str, expected_outputs: list[dict], max_iterations: int, + reasoning_effort: str | None = None, ) -> MetricRunResult: """Drive one full multi-turn metric-skill conversation and evaluate the result. @@ -195,7 +196,7 @@ def _execute_single_metric_run( try: for _iteration in range(max_iterations): turns += 1 - chat_result = client.send_message(conversation_id, current_question) + chat_result = client.send_message(conversation_id, current_question, reasoning_effort=reasoning_effort) candidate = _extract_metric_result(chat_result.tool_call_events or []) if candidate is not None: metric_result = candidate @@ -232,6 +233,7 @@ def run_agentic_metric_skill( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + reasoning_effort: str | None = None, ) -> AgenticMetricSummary: """Run the metric-skill agentic evaluation K times and return a summary. @@ -248,7 +250,7 @@ def run_agentic_metric_skill( try: run_results.append( _execute_single_metric_run( - client, sdk, workspace_id, conv_id_0, question, expected_outputs, max_iterations + client, sdk, workspace_id, conv_id_0, question, expected_outputs, max_iterations, reasoning_effort ) ) finally: @@ -260,7 +262,7 @@ def run_agentic_metric_skill( try: run_results.append( _execute_single_metric_run( - client, sdk, workspace_id, conv_id, question, expected_outputs, max_iterations + client, sdk, workspace_id, conv_id, question, expected_outputs, max_iterations, reasoning_effort ) ) finally: @@ -300,6 +302,7 @@ def evaluate_agentic_metric_skill( run_timestamp: str | None = None, model_version_override: str | None = None, run_metadata_extra: dict | None = None, + reasoning_effort: str | None = None, ) -> None: """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -319,6 +322,7 @@ def evaluate_agentic_metric_skill( k=k, max_iterations=max_iterations, initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py index cdbce48ab..a5ec633fd 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py @@ -67,6 +67,7 @@ def run_agentic_search_tool( expected_tool_call: dict, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + reasoning_effort: str | None = None, ) -> AgenticSearchSummary: """Run the search-tool agentic evaluation K times (single-turn each).""" run_results: list[SearchResult] = [] @@ -75,7 +76,7 @@ def run_agentic_search_tool( try: conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() try: - chat_result = client.send_message(conv_id_0, question) + chat_result = client.send_message(conv_id_0, question, reasoning_effort=reasoning_effort) tcs = chat_result.tool_call_events or [] selected = _tool_selection(tcs) correct = selected and _tool_correctness(tcs, expected_tool_call) @@ -94,7 +95,7 @@ def run_agentic_search_tool( for _ in range(1, k): conv_id = client.create_conversation() try: - chat_result = client.send_message(conv_id, question) + chat_result = client.send_message(conv_id, question, reasoning_effort=reasoning_effort) tcs = chat_result.tool_call_events or [] selected = _tool_selection(tcs) correct = selected and _tool_correctness(tcs, expected_tool_call) @@ -144,6 +145,7 @@ def evaluate_agentic_search_tool( run_timestamp: str | None = None, model_version_override: str | None = None, run_metadata_extra: dict | None = None, + reasoning_effort: str | None = None, ) -> None: """Run search-tool evaluation, log to Langfuse, and raise SearchToolAssertionError on failure.""" from datetime import datetime as _dt # noqa: PLC0415 @@ -162,6 +164,7 @@ def evaluate_agentic_search_tool( expected_tool_call=expected_tool_call, k=k, initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index 12914157a..ae1a1ec3e 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -152,6 +152,7 @@ def _execute_single_run( question: str, expected_outputs: list[CreatedVisualization], max_iterations: int = _DEFAULT_MAX_ITERATIONS, + reasoning_effort: str | None = None, ) -> RunResult: """Drive one full multi-turn conversation and evaluate the result.""" total_turns = 0.0 @@ -159,7 +160,7 @@ def _execute_single_run( all_tool_call_events: list[ToolCallEvent] = [] simulated_response_guide = expected_outputs[0] # primary candidate guides the simulated user - current_result = client.send_message(conversation_id, question) + current_result = client.send_message(conversation_id, question, reasoning_effort=reasoning_effort) for iteration in range(max_iterations): total_turns += 1.0 @@ -175,7 +176,7 @@ def _execute_single_run( break follow_up = generate_simulated_response(current_result.text_response, simulated_response_guide) - current_result = client.send_message(conversation_id, follow_up) + current_result = client.send_message(conversation_id, follow_up, reasoning_effort=reasoning_effort) skill_activated = _check_visualization_skill_activated(all_tool_call_events) actual_output: CreatedVisualization | None = None @@ -203,6 +204,7 @@ def run_agentic_visualization( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + reasoning_effort: str | None = None, ) -> AgenticRunSummary: """Run K independent conversations and return evaluation results. @@ -217,7 +219,9 @@ def run_agentic_visualization( try: conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() try: - run_results.append(_execute_single_run(client, conv_id_0, question, expected_outputs, max_iterations)) + run_results.append( + _execute_single_run(client, conv_id_0, question, expected_outputs, max_iterations, reasoning_effort) + ) finally: if initial_conversation_id is None: client.delete_conversation(conv_id_0) @@ -225,7 +229,9 @@ def run_agentic_visualization( for _ in range(1, k): conv_id = client.create_conversation() try: - run_results.append(_execute_single_run(client, conv_id, question, expected_outputs, max_iterations)) + run_results.append( + _execute_single_run(client, conv_id, question, expected_outputs, max_iterations, reasoning_effort) + ) finally: client.delete_conversation(conv_id) finally: @@ -265,6 +271,7 @@ def evaluate_agentic_visualization( model_version_override: str | None = None, run_metadata_extra: dict | None = None, record_output_path: str | None = None, + reasoning_effort: str | None = None, ) -> None: """Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure.""" import json as _json # noqa: PLC0415 @@ -285,6 +292,7 @@ def evaluate_agentic_visualization( k=k, max_iterations=max_iterations, initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index 091436b27..70d4b0984 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -268,10 +268,12 @@ def delete_conversation(self, conversation_id: str) -> None: except httpx.HTTPError: pass # best-effort cleanup - def send_message(self, conversation_id: str, question: str) -> ChatResult: + def send_message(self, conversation_id: str, question: str, *, reasoning_effort: str | None = None) -> ChatResult: url = f"{self._base}/{conversation_id}/messages" headers = {**self._auth, "Accept": "text/event-stream", "Content-Type": "application/json"} - body = {"item": {"role": "user", "content": {"type": "text", "text": question}}} + body: dict[str, Any] = {"item": {"role": "user", "content": {"type": "text", "text": question}}} + if reasoning_effort: + body["options"] = {"reasoningEffort": reasoning_effort} def _do() -> ChatResult: with self._client.stream("POST", url, json=body, headers=headers) as resp: @@ -280,7 +282,7 @@ def _do() -> ChatResult: return _retry_transient(_do, is_retryable=_is_retryable_exc) - def ask(self, item: DatasetItem) -> ChatResult: + def ask(self, item: DatasetItem, *, reasoning_effort: str | None = None) -> ChatResult: """Run one conversation: create, send, parse, clean up. The conversation_id is attached to the returned ChatResult for tracing. @@ -291,7 +293,7 @@ def ask(self, item: DatasetItem) -> ChatResult: conversation_id = self.create_conversation() success = False try: - result = self.send_message(conversation_id, item.question) + result = self.send_message(conversation_id, item.question, reasoning_effort=reasoning_effort) result.conversation_id = conversation_id success = True return result diff --git a/packages/gooddata-eval/src/gooddata_eval/core/config.py b/packages/gooddata-eval/src/gooddata_eval/core/config.py index 277785176..c940693b0 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/config.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/config.py @@ -20,3 +20,4 @@ class RunConfig: quiet: bool = False kind: str = "visualization" preserve_failed: bool = False + reasoning_effort: str | None = None diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py new file mode 100644 index 000000000..5c7368d79 --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -0,0 +1,67 @@ +# (C) 2026 GoodData Corporation +import pytest +from gooddata_eval.cli import agentic_runner as runner_mod +from gooddata_eval.cli.agentic_runner import _dispatch_agentic +from gooddata_eval.core.models import DatasetItem + +_MINIMAL_EXPECTED_OUTPUT = { + "vis_agentic": {"visualization": {"id": "v1", "type": "column_chart", "query": {"fields": {}}}}, + "agentic_visualization": {"visualization": {"id": "v1", "type": "column_chart", "query": {"fields": {}}}}, + "agentic_metric_skill": {}, + "agentic_alert_skill": {}, + "agentic_search": {}, + "agentic_general_question": "some expected text", + "agentic_guardrail": "some expected text", + "agentic_conversation": {"id": "conv1", "expected_skills": [], "turns": []}, +} + +_EVALUATE_FN_NAME = { + "vis_agentic": "evaluate_agentic_visualization", + "agentic_visualization": "evaluate_agentic_visualization", + "agentic_metric_skill": "evaluate_agentic_metric_skill", + "agentic_alert_skill": "evaluate_agentic_alert_skill", + "agentic_search": "evaluate_agentic_search_tool", + "agentic_general_question": "evaluate_agentic_general_question", + "agentic_guardrail": "evaluate_agentic_guardrail", + "agentic_conversation": "evaluate_agentic_conversation", +} + + +@pytest.mark.parametrize("kind", sorted(_MINIMAL_EXPECTED_OUTPUT)) +def test_dispatch_agentic_forwards_reasoning_effort(monkeypatch, kind): + """Every agentic kind must forward reasoning_effort to its evaluate_agentic_* function.""" + captured: dict = {} + + def _fake_evaluate(**kwargs): + captured.update(kwargs) + + monkeypatch.setattr(runner_mod, _EVALUATE_FN_NAME[kind], _fake_evaluate) + + item = DatasetItem( + id="i1", + dataset_name="d", + test_kind=kind, + question="q", + expected_output=_MINIMAL_EXPECTED_OUTPUT[kind], + ) + + _dispatch_agentic(item, "https://h", "tok", "ws", 1, None, "run-ts", None, reasoning_effort="HIGH") + + assert captured.get("reasoning_effort") == "HIGH" + + +def test_dispatch_agentic_defaults_reasoning_effort_to_none(monkeypatch): + captured: dict = {} + + def _fake_evaluate(**kwargs): + captured.update(kwargs) + + monkeypatch.setattr(runner_mod, "evaluate_agentic_guardrail", _fake_evaluate) + + item = DatasetItem( + id="i1", dataset_name="d", test_kind="agentic_guardrail", question="q", expected_output="expected" + ) + + _dispatch_agentic(item, "https://h", "tok", "ws", 1, None, "run-ts", None) + + assert captured.get("reasoning_effort") is None diff --git a/packages/gooddata-eval/tests/test_agentic_visualization.py b/packages/gooddata-eval/tests/test_agentic_visualization.py index cbeae6b3f..b10f90190 100644 --- a/packages/gooddata-eval/tests/test_agentic_visualization.py +++ b/packages/gooddata-eval/tests/test_agentic_visualization.py @@ -71,7 +71,7 @@ def test_execute_single_run_viz_on_first_turn(): assert result.total_turns == 1.0 assert result.total_steps == 2.0 assert result.conversation_id == "conv-1" - client.send_message.assert_called_once_with("conv-1", "Show revenue") + client.send_message.assert_called_once_with("conv-1", "Show revenue", reasoning_effort=None) def test_execute_single_run_clarification_then_viz(monkeypatch): @@ -92,7 +92,7 @@ def test_execute_single_run_clarification_then_viz(monkeypatch): assert result.eval_result.visualization_created is True assert result.total_turns == 2.0 assert client.send_message.call_count == 2 - assert client.send_message.call_args_list[1] == call("conv-1", "Revenue please") + assert client.send_message.call_args_list[1] == call("conv-1", "Revenue please", reasoning_effort=None) def test_execute_single_run_no_viz_no_text(): @@ -148,7 +148,7 @@ def test_run_agentic_visualization_uses_initial_conversation_for_run_0(): # create_conversation should NOT be called for run 0 instance.create_conversation.assert_not_called() - instance.send_message.assert_called_once_with("existing-conv", "Show revenue") + instance.send_message.assert_called_once_with("existing-conv", "Show revenue", reasoning_effort=None) # the caller-supplied conversation is left intact; the function only deletes conversations it created instance.delete_conversation.assert_not_called() assert len(summary.run_results) == 1 diff --git a/packages/gooddata-eval/tests/test_cli.py b/packages/gooddata-eval/tests/test_cli.py index 28aecb3c4..5a549cfb1 100644 --- a/packages/gooddata-eval/tests/test_cli.py +++ b/packages/gooddata-eval/tests/test_cli.py @@ -535,8 +535,6 @@ def close(self): ... monkeypatch.setattr(cli_main, "WorkspaceModelController", _FakeController) - original_chat_client = cli_main.ChatClient - def _capture_chat_client(**kwargs): captured_kwargs.update(kwargs) return object() @@ -573,6 +571,161 @@ def _fake_run(items, backend, *, runs, model, workspace_id, **kw): assert captured_kwargs.get("preserve_failed") is True +def _reasoning_effort_harness(monkeypatch, fixtures_dir): + """Shared scaffolding for --reasoning-effort tests: fakes everything but _RoutingBackend.""" + monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok")) + captured: dict = {} + + class _FakeController: + def __init__(self, *a, **k): ... + def get_active(self): + return ActiveLlmProvider(provider_id="p", default_model_id="gpt-5.2") + + def resolve_and_activate(self, requested, provider=None): + return ResolvedModel(provider_id="p", model_id="gpt-5.2", switched=False, provider_name="P") + + def restore(self, original): ... + def close(self): ... + + monkeypatch.setattr(cli_main, "WorkspaceModelController", _FakeController) + monkeypatch.setattr(cli_main, "ChatClient", lambda **kwargs: object()) + monkeypatch.setattr(cli_main, "SummaryClient", lambda **kwargs: object()) + + class _FakeBackend: + def __init__(self, chat, summary, *, reasoning_effort=None): + captured["reasoning_effort"] = reasoning_effort + + def ask(self, item): + raise AssertionError("not used by the fake run_items") + + def close(self): ... + + monkeypatch.setattr(cli_main, "_RoutingBackend", _FakeBackend) + + def _fake_run(items, backend, *, runs, model, workspace_id, **kw): + return EvalReport( + model=model, + workspace_id=workspace_id, + items=[ + ItemReport(id="i1", dataset_name="d", test_kind="visualization", question="q", pass_at_k=True, runs=1) + ], + ) + + monkeypatch.setattr(cli_main, "run_items", _fake_run) + return captured + + +def test_cli_reasoning_effort_flag_parsed(monkeypatch, fixtures_dir): + """--reasoning-effort threads its value into _RoutingBackend.""" + captured = _reasoning_effort_harness(monkeypatch, fixtures_dir) + + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--reasoning-effort", + "HIGH", + "--quiet", + ] + ) + assert exit_code == 0 + assert captured.get("reasoning_effort") == "HIGH" + + +def test_cli_reasoning_effort_omitted_by_default(monkeypatch, fixtures_dir): + """Without --reasoning-effort or the env var, no effort is requested.""" + monkeypatch.delenv("GD_EVAL_REASONING_EFFORT", raising=False) + captured = _reasoning_effort_harness(monkeypatch, fixtures_dir) + + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--quiet", + ] + ) + assert exit_code == 0 + assert captured.get("reasoning_effort") is None + + +def test_cli_reasoning_effort_env_var_fallback(monkeypatch, fixtures_dir): + """GD_EVAL_REASONING_EFFORT is used when --reasoning-effort is not passed.""" + monkeypatch.setenv("GD_EVAL_REASONING_EFFORT", "LOW") + captured = _reasoning_effort_harness(monkeypatch, fixtures_dir) + + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--quiet", + ] + ) + assert exit_code == 0 + assert captured.get("reasoning_effort") == "LOW" + + +def test_cli_reasoning_effort_flag_wins_over_env_var(monkeypatch, fixtures_dir): + """--reasoning-effort takes precedence over GD_EVAL_REASONING_EFFORT when both are set.""" + monkeypatch.setenv("GD_EVAL_REASONING_EFFORT", "LOW") + captured = _reasoning_effort_harness(monkeypatch, fixtures_dir) + + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--reasoning-effort", + "HIGH", + "--quiet", + ] + ) + assert exit_code == 0 + assert captured.get("reasoning_effort") == "HIGH" + + +def test_cli_reasoning_effort_rejects_invalid_value(fixtures_dir): + """argparse rejects any value outside LOW/MEDIUM/HIGH.""" + with pytest.raises(SystemExit): + cli_main.parse_args( + [ + "run", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--reasoning-effort", + "EXTREME", + ] + ) + + def test_cli_rejects_negative_concurrency(monkeypatch, fixtures_dir): monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok")) exit_code = cli_main.main( diff --git a/packages/gooddata-eval/tests/test_sse_client.py b/packages/gooddata-eval/tests/test_sse_client.py index c5590e428..5717d8d1b 100644 --- a/packages/gooddata-eval/tests/test_sse_client.py +++ b/packages/gooddata-eval/tests/test_sse_client.py @@ -205,6 +205,30 @@ def handler(request): assert sleeps == [] +def test_send_message_omits_reasoning_effort_by_default(): + requests = [] + + def handler(request): + requests.append(json.loads(request.content)) + return httpx.Response(200, content=_OK_SSE) + + client = _client_with_handler(handler) + client.send_message("conv", "q") + assert "options" not in requests[0] + + +def test_send_message_sends_reasoning_effort_when_given(): + requests = [] + + def handler(request): + requests.append(json.loads(request.content)) + return httpx.Response(200, content=_OK_SSE) + + client = _client_with_handler(handler) + client.send_message("conv", "q", reasoning_effort="HIGH") + assert requests[0]["options"] == {"reasoningEffort": "HIGH"} + + def test_create_conversation_retries_then_succeeds(monkeypatch): sleeps = [] monkeypatch.setattr(sse_mod.time, "sleep", lambda s: sleeps.append(s)) From 7fdb4701bc9b30d28b8d45ec8bc7a6130873f6c2 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Tue, 4 Aug 2026 10:39:28 +0200 Subject: [PATCH 2/2] fix(gooddata-eval): validate GD_EVAL_REASONING_EFFORT the same as --reasoning-effort The argparse choices constraint only covered the CLI flag; an invalid env var value slipped straight through to ChatClient/the API. Reject it in main() before building RunConfig. Addresses CodeRabbit review comment on PR #1704. --- .../src/gooddata_eval/cli/main.py | 14 +++++++++++- packages/gooddata-eval/tests/test_cli.py | 22 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/main.py b/packages/gooddata-eval/src/gooddata_eval/cli/main.py index 8f08d169f..57a5b44e6 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/main.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/main.py @@ -425,11 +425,23 @@ def on_langfuse_item_done( return _EXIT_OK +_VALID_REASONING_EFFORTS = frozenset({"LOW", "MEDIUM", "HIGH"}) + + def main(argv: list[str] | None = None) -> int: args = parse_args(argv if argv is not None else sys.argv[1:]) if hasattr(args, "concurrency") and args.concurrency < 1: print("error: --concurrency must be >= 1.", file=sys.stderr) return _EXIT_OPERATIONAL_ERROR + reasoning_effort = None + if hasattr(args, "reasoning_effort"): + reasoning_effort = args.reasoning_effort or os.environ.get("GD_EVAL_REASONING_EFFORT") + if reasoning_effort is not None and reasoning_effort not in _VALID_REASONING_EFFORTS: + print( + f"error: reasoning effort must be one of {sorted(_VALID_REASONING_EFFORTS)}, got {reasoning_effort!r}.", + file=sys.stderr, + ) + return _EXIT_OPERATIONAL_ERROR try: host, token = resolve_connection(host=args.host, token=args.token, profile=args.profile) if args.command == "models": @@ -448,7 +460,7 @@ def main(argv: list[str] | None = None) -> int: quiet=args.quiet, kind=args.kind, preserve_failed=args.preserve_failed, - reasoning_effort=args.reasoning_effort or os.environ.get("GD_EVAL_REASONING_EFFORT"), + reasoning_effort=reasoning_effort, ) return _run(config) except ( diff --git a/packages/gooddata-eval/tests/test_cli.py b/packages/gooddata-eval/tests/test_cli.py index 5a549cfb1..525d3bfcc 100644 --- a/packages/gooddata-eval/tests/test_cli.py +++ b/packages/gooddata-eval/tests/test_cli.py @@ -726,6 +726,28 @@ def test_cli_reasoning_effort_rejects_invalid_value(fixtures_dir): ) +def test_cli_reasoning_effort_rejects_invalid_env_var(monkeypatch, fixtures_dir): + """GD_EVAL_REASONING_EFFORT bypasses argparse choices, so main() must validate it too.""" + monkeypatch.setenv("GD_EVAL_REASONING_EFFORT", "EXTREME") + monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok")) + + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--quiet", + ] + ) + assert exit_code == cli_main._EXIT_OPERATIONAL_ERROR + + def test_cli_rejects_negative_concurrency(monkeypatch, fixtures_dir): monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok")) exit_code = cli_main.main(