diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index a69fabca1246..4f7775588b05 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -8,6 +8,14 @@ - `QAEvaluator` accepts a Decision model configuration and passes `credential` and the Decision execution controls to its model-based evaluators, which run on separate threads, so the credential must be synchronous. With a chat model they are not passed, as before. - The binary evaluators (task adherence, task completion, tool call success, tool input accuracy, tool output utilization and tool selection) accept a `probability_threshold` keyword. For Decision models it sets the probability at or above which the score is 1 (default 0.75) and overrides the value in `model_config`; chat models ignore it with a warning. - Conversation inputs are handled as before with a Decision model: evaluators that accept them score each user and assistant turn separately and aggregate the results. The catalog's whole-conversation rubrics are not included. +- A failed Decision request names a recognized reason in `decision_model.error_kind` of the error's properties, which is kept on failed result rows, and in the error message, for example `The decision model returned HTTP 422 (context_window_exceeded).` The provider states these reasons for an HTTP 422 only in text, which is never exposed: `context_window_exceeded` when the evidence and one question exceed the model's context window, `request_token_limit_exceeded` when the evidence and all questions exceed the model's request limit, `question_limit_exceeded` and `invalid_question`. A request over `max_request_bytes` or `max_state_question_bytes`, which is not sent, has the kind `request_bytes_exceeded`. + +### Bugs Fixed + +- `_ToolCallSuccessEvaluator` no longer scores successful tool calls 0 when a chat judge model, such as gpt-4.1 or gpt-4o, puts `score` and `status` inside `properties`. The example outputs in its prompt are now valid JSON, and `score`, `status` and `reason` are read from `properties` when the output has none at the top level. A missing score is no longer reported as 0, and any score other than 0 or 1 raises a system-classified `EvaluationException`, as `_ToolInputAccuracyEvaluator` does. +- `_ToolCallSuccessEvaluator` reports a list of messages that holds no tool calls, such as one that contains only text or only MCP approval messages, as not applicable without calling the chat or Decision judge model. A `tool_calls`, `function_call` or `tool_call_id` field that is None or empty, as in OpenAI's `model_dump()` of a reply without tool calls, does not count as a tool call. Strings, text that contains `[TOOL_CALL]` or `[TOOL_RESULT]`, and messages with any other content are still sent to the judge model. +- `evaluate()` computes `.binary_aggregate` over the rows whose result is `pass` or `fail`. Other rows, such as not applicable or errored rows, used to count as failures, although the mean score leaves them out. An evaluator without `pass` or `fail` rows, including a custom evaluator whose results have other values, has no `binary_aggregate`, as one without scores has no mean. +- An evaluator's inputs keep the order in which its `__call__` overloads declare them, instead of the order of a Python `set`, which varies with the process's hash seed. An evaluator that sends these inputs to a Decision model unchanged, as several Microsoft Foundry catalog evaluators do, could list the same fields in a different order in each process, and the Decision model could answer the same input differently. The built-in evaluators in this package already send their Decision inputs in a fixed order. ## 1.18.10 (2026-10-22) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py index debf679e61db..63f1d9960485 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py @@ -22,7 +22,7 @@ from ..._evaluate._utils import _has_aggregator from ..._constants import Prefixes, PF_BATCH_TIMEOUT_SEC from ..._exceptions import EvaluationException -from ..._evaluators._common._model_runners import _is_finite_number, validated_execution +from ..._evaluators._common._model_runners import DECISION_ERROR_KINDS, _is_finite_number, validated_execution from .._utils import get_int_env_var as get_int @@ -56,6 +56,7 @@ def _decision_error_metadata(exception: EvaluationException) -> Dict[str, Any]: ("backend", ("foundry_decision", "system_one")), ("protocol", ("shared_sdk", "system_one")), ("result_usage_scope", ("reported_attempts_only", "final_response_only", "reported_turns_only")), + ("error_kind", DECISION_ERROR_KINDS), ): value = native.get(key) if isinstance(value, str) and value in accepted: diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py index d432380653e9..61c05f398fb8 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py @@ -273,7 +273,10 @@ def _aggregation_binary_output(df: pd.DataFrame) -> Dict[str, float]: """ Aggregate binary output results (pass/fail) from evaluation dataframe. - For each evaluator, calculates the proportion of "pass" results. + For each evaluator, calculates the proportion of "pass" results among the rows whose result is "pass" or "fail". + Other rows, such as not applicable or errored rows, are left out, as they are from the mean score, and a result + column without "pass" or "fail" rows gives no binary aggregate. An evaluator with several result columns reports + its last one. :param df: The dataframe of evaluation results. :type df: ~pandas.DataFrame @@ -312,14 +315,16 @@ def _aggregation_binary_output(df: pd.DataFrame) -> Dict[str, float]: ) continue - # Calculate the proportion of EVALUATION_PASS_FAIL_MAPPING[True] results - total_rows = len(df) pass_count = value_counts.get(EVALUATION_PASS_FAIL_MAPPING[True], 0) - proportion = pass_count / total_rows if total_rows > 0 else 0.0 + judged_rows = pass_count + value_counts.get(EVALUATION_PASS_FAIL_MAPPING[False], 0) # Set the result with the evaluator name as the key result_key = f"{evaluator_name}.{BINARY_AGGREGATE_SUFFIX}" - results[result_key] = round(proportion, 2) + if judged_rows == 0: + # The evaluator's last result column decides, so an earlier column's rate is not kept. + results.pop(result_key, None) + continue + results[result_key] = round(pass_count / judged_rows, 2) return results diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py index db3feaa8613d..7b9e46ed9934 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py @@ -249,13 +249,14 @@ def _get_matching_overload_inputs(self, **kwargs) -> List[str]: def _get_all_singleton_inputs(self) -> List[str]: """Get a flattened list of all possible singleton inputs across all overloads. + Names keep the order in which the overloads declare them, so an evaluator's inputs, and the state of a decision + request that sends them unchanged, have the same order in every process. A set's order would vary with the + process's hash seed. + :return: Flattened list of all singleton input names. :rtype: List[str] """ - all_inputs = set() - for inputs in self._singleton_inputs: - all_inputs.update(inputs) - return list(all_inputs) + return list(dict.fromkeys(name for inputs in self._singleton_inputs for name in inputs)) def _derive_conversation_converter( self, diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py index e2d34742c296..b2035dda5695 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py @@ -45,7 +45,6 @@ def value(self) -> str: DECISION_MODEL_TYPE = "decision" _DECISION_PATH = "/providers/microsoft/v1/systemone" _DECISION_TOKEN_SCOPE = "https://cognitiveservices.azure.com/.default" -_DECISION_RETRY_STATUS_CODES = [408, 429, 500, 502, 503, 504, 529] LOGGER = logging.getLogger(__name__) _PREFIX = "azure.ai.evaluation.decision." _DECISION_CONFIG_KEYS = { @@ -59,6 +58,29 @@ def value(self) -> str: _DEFAULT_PROBABILITY_THRESHOLD = 0.75 # The service adds api_version to every resolved model configuration; the decision endpoint does not use it. _IGNORED_CONFIG_KEYS = {"api_version"} +_ERROR_CODE = re.compile(r"[A-Za-z][A-Za-z0-9_.-]{0,127}") +# A 422 states its reason only in a ``detail`` sentence, which is never surfaced. These reasons are recognized by the +# provider's fixed sentence openings, so a quoted question identifier later in the sentence cannot change the kind, and +# reported as a bounded ``decision_model.error_kind``; any other sentence is ignored. Only the sentence's first +# characters are read, which hold every opening, so a long question identifier cannot hide the reason. +_PROVIDER_DETAIL_KINDS = ( + ( + re.compile(r"invalid model request: Question prompt exceeds the \d+-token context limit\b"), + "context_window_exceeded", + ), + ( + re.compile( + r"invalid TypeSafe request: request is about \d+ tokens; the limit for the state plus all questions is \d+" + ), + "request_token_limit_exceeded", + ), + (re.compile(r"invalid TypeSafe request: request has \d+ questions; the limit is \d+"), "question_limit_exceeded"), + (re.compile(r"invalid TypeSafe request: invalid [a-z_]+ for question \""), "invalid_question"), +) +_MAX_PROVIDER_DETAIL_CHARS = 1024 +# Reported when a request exceeds the caller's max_request_bytes or max_state_question_bytes and is not sent. +_REQUEST_BYTES_EXCEEDED = "request_bytes_exceeded" +DECISION_ERROR_KINDS = tuple(kind for _, kind in _PROVIDER_DETAIL_KINDS) + (_REQUEST_BYTES_EXCEEDED,) def is_decision_model_config(model_config: Any) -> bool: @@ -340,6 +362,26 @@ def _is_finite_number(value: Any) -> bool: return False +def _provider_error_code(payload: Any) -> Optional[str]: + """Return the provider's nested ``error.code`` when it is a bounded identifier, otherwise None.""" + provider_error = payload.get("error") if isinstance(payload, dict) else None + code = provider_error.get("code") if isinstance(provider_error, dict) else None + return code if isinstance(code, str) and _ERROR_CODE.fullmatch(code) is not None else None + + +def _provider_error_kind(status: int, payload: Any) -> Optional[str]: + """Name a 422 that states its reason only in a ``detail`` sentence, from the sentence's fixed opening. + + For example ``{"detail": "invalid model request: Question prompt exceeds the 32768-token context limit"}`` is + ``context_window_exceeded``. Any other status or unrecognized sentence returns None and is never surfaced. + """ + detail = payload.get("detail") if status == 422 and isinstance(payload, dict) else None + if not isinstance(detail, str): + return None + opening = detail[:_MAX_PROVIDER_DETAIL_CHARS] + return next((kind for phrase, kind in _PROVIDER_DETAIL_KINDS if phrase.match(opening)), None) + + def _probability(value: Any) -> float: if not _is_finite_number(value): raise ValueError("Decision probabilities must be finite numbers.") @@ -415,6 +457,10 @@ class DecisionModelRunner: it stops awaiting synchronous credentials but cannot stop their worker threads. Byte limits measure compact unescaped UTF-8 JSON, with ``max_state_question_bytes`` applying to the largest individual pair. ``max_response_bytes`` limits acceptance before parsing, not transport allocation. + A failed request names a recognized reason in ``decision_model.error_kind`` of the error's properties: + ``context_window_exceeded`` when the evidence and one question exceed the model's context window, + ``request_token_limit_exceeded``, ``question_limit_exceeded`` or ``invalid_question`` for the provider's other + rejections, and ``request_bytes_exceeded`` for a request over the byte limits, which is not sent. """ def __init__( @@ -583,6 +629,7 @@ async def __call__(self, *, state: Any, questions: Dict[str, Dict[str, Any]], ti if (self._max_request_bytes is not None and len(content) > self._max_request_bytes) or ( self._max_state_question_bytes is not None and state_question_bytes > self._max_state_question_bytes ): + properties["decision_model"]["error_kind"] = _REQUEST_BYTES_EXCEEDED raise _failed("The decision request exceeds the configured byte budget.", ErrorBlame.USER_ERROR) try: payload = await asyncio.wait_for( @@ -670,13 +717,14 @@ async def _post(self, content: bytes, *, deadline: float, execution: DecisionExe policies: Dict[str, Any] = { "custom_hook_policy": execution, + # azure-core retries 408, 429, 500, 502, 503, 504 and 506 through 998 by default; POST is opted in + # per request below. "retry_policy": AsyncRetryPolicy( retry_total=self._max_retries, retry_connect=self._max_retries, retry_read=self._max_retries, retry_status=self._max_retries, retry_backoff_factor=0.8, - retry_on_status_codes=_DECISION_RETRY_STATUS_CODES, ), "redirect_policy": AsyncRedirectPolicy(permit_redirects=False), } @@ -719,15 +767,13 @@ async def _post(self, content: bytes, *, deadline: float, execution: DecisionExe payload = None if status != 200: blame = ErrorBlame.USER_ERROR if status in (400, 401, 403, 404, 422) else ErrorBlame.SYSTEM_ERROR - provider_error = payload.get("error") if isinstance(payload, dict) else None - code = provider_error.get("code") if isinstance(provider_error, dict) else None - diagnostic = ( - f" ({code})" - if isinstance(code, str) and re.fullmatch(r"[A-Za-z][A-Za-z0-9_.-]{0,127}", code) is not None - else "" - ) + code = _provider_error_code(payload) + kind = _provider_error_kind(status, payload) + diagnostic = f" ({code or kind})" if code or kind else "" error = _failed(f"The decision model returned HTTP {status}{diagnostic}.", blame) error.properties = self._response_properties(payload) + if kind is not None: + error.properties["decision_model"]["error_kind"] = kind raise error return payload diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py index b098674aa189..7ecd702a7a8b 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py @@ -4,7 +4,8 @@ import math import os import logging -from typing import Dict, Union, List, Optional +import reprlib +from typing import Any, Dict, Union, List, Optional from typing_extensions import overload, override from azure.ai.evaluation._exceptions import ( EvaluationException, @@ -20,6 +21,14 @@ logger = logging.getLogger(__name__) +_NO_TOOL_CALLS_MESSAGE = "No tool calls found in response." +# Message content without tool activity. Any other content type may be a tool call or result. +_TEXT_CONTENT_TYPES = frozenset({"text", "input_text", "output_text"}) +_CONVERSATION_ROLES = frozenset({"system", "developer", "user", "assistant"}) +_TOOL_CALL_KEYS = ("tool_calls", "function_call") +_TOOL_TRACE_MARKERS = ("[TOOL_CALL]", "[TOOL_RESULT]") +_APPROVAL_TYPES = frozenset({"mcp_approval_request", "mcp_approval_response"}) + @experimental class _ToolCallSuccessEvaluator(PromptyEvaluatorBase[Union[str, float]]): @@ -131,11 +140,19 @@ def __call__( Example with list of messages: evaluator = _ToolCallSuccessEvaluator(model_config) - response = [{'createdAt': 1700000070, 'run_id': '0', 'role': 'assistant', - 'content': [{'type': 'text', 'text': '**Day 1:** Morning: Visit Louvre Museum (9 AM - 12 PM)...'}]}] + response = [ + {'role': 'assistant', 'content': [{'type': 'tool_call', 'tool_call_id': 'call_1', + 'name': 'fetch_weather', 'arguments': {'location': 'Paris'}}]}, + {'role': 'tool', 'tool_call_id': 'call_1', + 'content': [{'type': 'tool_result', 'tool_result': {'temperature': '20C'}}]}, + {'role': 'assistant', 'content': [{'type': 'text', 'text': 'It is 20C in Paris.'}]}] result = evaluator(response=response, ) + A list of messages that holds only text has no tool calls, so it is reported as not applicable without + calling the judge model. A tool_calls, function_call or tool_call_id field that is None or empty does not + count as a tool call. + :keyword response: The response being evaluated, either a string or a list of messages (full agent response potentially including tool calls) :paramtype response: Union[str, List[dict]] @@ -197,6 +214,9 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[str, float]]: # t ) if isinstance(eval_input.get("response"), list): + # Decided before preprocessing, which drops whole messages that hold an MCP approval. + if _has_no_tool_calls(eval_input["response"]): + return self._return_not_applicable_result(_NO_TOOL_CALLS_MESSAGE, self._threshold) eval_input["response"] = _preprocess_messages(eval_input["response"]) eval_input["tool_calls"] = _reformat_tool_calls_results(eval_input["response"], logger) # If response is a string, pass directly without reformatting @@ -235,17 +255,25 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[str, float]]: # t llm_output = prompty_output_dict.get("llm_output", prompty_output_dict) if isinstance(llm_output, dict): + properties = llm_output.get("properties") + llm_properties = properties if isinstance(properties, dict) else {} + reason = _top_level_or_nested(llm_output, llm_properties, "reason", "") # Handle skipped status from LLM - llm_status = llm_output.get("status", "completed") + llm_status = _top_level_or_nested(llm_output, llm_properties, "status", "completed") if llm_status == "skipped": - reason = llm_output.get("reason", "") return self._return_not_applicable_result(reason, self._threshold) - llm_properties = llm_output.get("properties", {}) or {} - - score = float(llm_output.get("score", 0)) + raw_score = _top_level_or_nested(llm_output, llm_properties, "score") + score = _binary_score(raw_score) + if score is None: + raise EvaluationException( + message=f"Invalid score value: {reprlib.repr(raw_score)}. Expected 0 or 1.", + internal_message="Invalid score value.", + blame=ErrorBlame.SYSTEM_ERROR, + category=ErrorCategory.FAILED_EXECUTION, + target=ErrorTarget.TOOL_CALL_SUCCESS_EVALUATOR, + ) success_result = "pass" if score >= 1.0 else "fail" - reason = llm_output.get("reason", "") llm_properties.update(self._get_token_metadata(prompty_output_dict)) return { self._result_key: score, @@ -265,6 +293,77 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[str, float]]: # t ) +def _top_level_or_nested(output: Dict, nested: Dict, key: str, default: Any = None) -> Any: + """Return the judge output's value for ``key``, or the one nested in its properties when it has none.""" + value = output.get(key) + return nested.get(key, default) if value is None else value + + +def _binary_score(value: Any) -> Optional[float]: + """Return a judge's score as 1.0 or 0.0, or None when it is neither.""" + try: + score = float(value) + except (TypeError, ValueError, OverflowError): + return None + if score == 1.0: + return 1.0 + if score == 0.0: + return 0.0 + return None + + +def _holds_nothing(value: Any) -> bool: + """Return True for None and an empty string, list or dict.""" + return value is None or (isinstance(value, (str, list, dict)) and not value) + + +def _is_one_of(value: Any, names: frozenset) -> bool: + """Return True when the value is one of the names; any other value, including an unhashable one, is not.""" + return isinstance(value, str) and value in names + + +def _has_no_tool_calls(messages) -> bool: + """Return True only for a list of text messages and MCP approvals that holds no tool calls, results or traces. + + A tool_calls, function_call or tool_call_id field that holds nothing, such as the None that OpenAI's model_dump() + writes, counts as absent. A list in any other format may hold tool activity this evaluator does not parse, so the + judge assesses it. + """ + if not isinstance(messages, list) or not messages: + return False + for message in messages: + if not isinstance(message, dict) or not all(_holds_nothing(message.get(key)) for key in _TOOL_CALL_KEYS): + return False + content = message.get("content") + items = content + if isinstance(content, list): + items = [ + item + for item in content + if not (isinstance(item, dict) and _is_one_of(item.get("type"), _APPROVAL_TYPES)) + ] + if content and not items: + # Only MCP approval requests or responses, which are not tool calls; a response names its request + # in tool_call_id. + continue + if not _is_one_of(message.get("role"), _CONVERSATION_ROLES) or not _holds_nothing(message.get("tool_call_id")): + return False + if isinstance(content, str): + texts = [content] + elif isinstance(items, list) and all( + isinstance(item, dict) and _is_one_of(item.get("type"), _TEXT_CONTENT_TYPES) for item in items + ): + texts = [item.get("text") for item in items] + else: + return False + # Text can carry tool calls in this evaluator's own trace format. + if not all( + isinstance(text, str) and not any(marker in text for marker in _TOOL_TRACE_MARKERS) for text in texts + ): + return False + return True + + def _filter_to_used_tools(tool_definitions, msgs_list, logger=None): """Filter the tool definitions to only include those that were actually used in the messages lists.""" try: diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/tool_call_success.prompty b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/tool_call_success.prompty index d7df87a2004d..f9af98623073 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/tool_call_success.prompty +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/tool_call_success.prompty @@ -94,7 +94,7 @@ EXPECTED OUTPUT { "reason": "None of the results indicate an error", "properties": { - "failed_tools": "", + "failed_tools": "" }, "score": 1, "status": "completed" @@ -111,7 +111,7 @@ EXPECTED OUTPUT { "reason": "None of the results indicate an error", "properties": { - "failed_tools": "", + "failed_tools": "" }, "score": 1, "status": "completed" @@ -128,7 +128,7 @@ EXPECTED OUTPUT { "reason": "Although the returned value 7 is not the square root of 4, this is a business mistake in the tool. The tool did not return a result indicating a technical error", "properties": { - "failed_tools": "", + "failed_tools": "" }, "score": 1, "status": "completed" @@ -146,7 +146,7 @@ EXPECTED OUTPUT { "reason": "The tool returned a semicolon separated list of names. Although the description in the definition says it should return comma-separated list , this formatting mistake is a business mistake of the tool, not a technical failure. The tool did not return an error", "properties": { - "failed_tools": "", + "failed_tools": "" }, "score": 1, "status": "completed" @@ -165,7 +165,7 @@ EXPECTED OUTPUT { "reason": "The tool returned empty response which is accepted given that this tool functionality does not include returning data to the caller", "properties": { - "failed_tools": "", + "failed_tools": "" }, "score": 1, "status": "completed" @@ -186,7 +186,7 @@ EXPECTED OUTPUT { "reason": "The tool returned empty response , however , given the tool definition , it should never return empty response because there should be weather info at any given point in time. An empty response here is considered a technical failure. The conclusion is the get_weather_info failed", "properties": { - "failed_tools": "get_weather_info", + "failed_tools": "get_weather_info" }, "score": 0, "status": "completed" @@ -203,7 +203,7 @@ EXPECTED OUTPUT { "reason": "The tool returned a string indicating that it failed", "properties": { - "failed_tools": "get_current_user_Info", + "failed_tools": "get_current_user_Info" }, "score": 0, "status": "completed" @@ -220,7 +220,7 @@ EXPECTED OUTPUT { "reason": "The tool returned an object with empty fields and a string indicating that it failed", "properties": { - "failed_tools": "get_current_user_Info", + "failed_tools": "get_current_user_Info" }, "score": 0, "status": "completed" @@ -237,7 +237,7 @@ EXPECTED OUTPUT { "reason": "The call for GetWeatherInfo returned an object containing single property 'temp' that is an empty string. This means the call to GetWeatherInfo returned empty result while weather info should be available at any time", "properties": { - "failed_tools": "GetWeatherInfo", + "failed_tools": "GetWeatherInfo" }, "score": 0, "status": "completed" @@ -254,7 +254,7 @@ EXPECTED OUTPUT { "reason": "the returned result indicates that the call to get_day_of_week timed out", "properties": { - "failed_tools": "get_day_of_week", + "failed_tools": "get_day_of_week" }, "score": 0, "status": "completed" @@ -272,7 +272,7 @@ EXPECTED OUTPUT { "reason": "null indicates an empty result which cannot be an accepted output of the tool given the tool definition since any given date represents a day of week", "properties": { - "failed_tools": "get_day_of_week", + "failed_tools": "get_day_of_week" }, "score": 0, "status": "completed" @@ -290,7 +290,7 @@ EXPECTED OUTPUT { "reason": "Empty object cannot be an accepted output of the tool given the tool definition since any given date should represent a day of week", "properties": { - "failed_tools": "get_day_of_week", + "failed_tools": "get_day_of_week" }, "score": 0, "status": "completed" @@ -309,7 +309,7 @@ EXPECTED OUTPUT { "reason": "GetWeatherInfo returned an empty response while it should return the weather info and BookTicket returned an error.Both tools failed.", "properties": { - "failed_tools": "GetWeatherInfo,BookTicket", + "failed_tools": "GetWeatherInfo,BookTicket" }, "score": 0, "status": "completed" @@ -328,7 +328,7 @@ EXPECTED OUTPUT { "reason": "Although GetWeatherInfo succeeded, BookTicket returned an error. The final result is failure because one of the tool calls has failed", "properties": { - "failed_tools": "BookTicket", + "failed_tools": "BookTicket" }, "score": 0, "status": "completed" diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py index b5ecfc6c3e72..49020631c5eb 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py @@ -222,6 +222,85 @@ def test_failed_metadata_rejects_untyped_and_unrecognized_provenance(self): ) assert _decision_error_metadata(error)["properties"] == {"decision_model": {}} + @pytest.mark.parametrize( + "kind,kept", + [ + ("context_window_exceeded", True), + ("request_token_limit_exceeded", True), + ("question_limit_exceeded", True), + ("invalid_question", True), + ("request_bytes_exceeded", True), + ("DeploymentNotFound", False), + ("context_window_exceeded\nprivate", False), + (42, False), + (None, False), + ], + ) + def test_failed_metadata_keeps_only_a_known_error_kind(self, kind, kept): + error = EvaluationException("synthetic failure", properties={"decision_model": {"error_kind": kind}}) + assert _decision_error_metadata(error)["properties"] == {"decision_model": {"error_kind": kind} if kept else {}} + + @pytest.mark.parametrize( + "detail,max_request_bytes,kind,message", + [ + ( + "invalid model request: Question prompt exceeds the 32768-token context limit including the decision " + "token", + None, + "context_window_exceeded", + "The decision model returned HTTP 422 (context_window_exceeded).", + ), + (None, 16, "request_bytes_exceeded", "The decision request exceeds the configured byte budget."), + ], + ids=["provider", "byte_limit"], + ) + def test_failure_reason_survives_batch_conversion( + self, batch_client, tmp_path, monkeypatch, detail, max_request_bytes, kind, message + ): + sent = [] + + class Response(HttpResponse): + def __init__(self, request): + super().__init__(request, None) + self.status_code = 422 + self.headers = {} + + def body(self): + return json.dumps({"detail": detail}).encode() + + class Transport(AsyncHttpTransport): + async def open(self): + pass + + async def close(self): + pass + + async def __aexit__(self, *_): + await self.close() + + async def send(self, request, **kwargs): + sent.append(request) + return Response(request) + + monkeypatch.setattr( + _model_runners, "get_async_http_client", lambda **_: AsyncHttpPipeline(transport=Transport()) + ) + + async def judge(query): + runner = DecisionModelRunner(CONFIG, max_retries=0, max_request_bytes=max_request_bytes) + return await strategy(runner).evaluate({"query": query}) + + _, rows = convert_batch(batch_client, judge, tmp_path) + result = next(item for item in rows[0]["results"] if item["name"] == "judge") + assert result["status"] == "error" and result["score"] is None + assert result["sample"]["error"]["blame"] == "UserError" + assert message in result["sample"]["error"]["message"] + native = result["properties"]["decision_model"] + assert native["error_kind"] == kind + assert native["execution"]["last_http_status"] == (422 if detail else None) + assert len(sent) == (1 if detail else 0) + assert "32768-token" not in json.dumps(rows) + @pytest.mark.parametrize("members", [5, 6]) @pytest.mark.parametrize("case", ["completed", "skipped", "mixed", "semantic_error"]) @pytest.mark.parametrize("primary_first", [False, True]) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index 49c2b863c71e..5761abc9e135 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -9,11 +9,14 @@ import json import logging import math +import os +import subprocess import sys import threading from copy import deepcopy from logging.handlers import BufferingHandler from concurrent.futures import ThreadPoolExecutor +from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock, Mock, call, patch @@ -23,7 +26,7 @@ from azure.core.pipeline.transport import AsyncHttpTransport, HttpResponse from azure.core.exceptions import ServiceRequestError -from azure.ai.evaluation import CoherenceEvaluator, FluencyEvaluator +from azure.ai.evaluation import CoherenceEvaluator, FluencyEvaluator, GroundednessEvaluator from azure.ai.evaluation._evaluators._common import EvaluatorBase, ModelEvaluatorBase, PromptyEvaluatorBase from azure.ai.evaluation._evaluators._common import _model_runners from azure.ai.evaluation._evaluators._common._decision_strategy import ( @@ -315,7 +318,8 @@ def test_bounded_retries_without_redirects(self, client): assert isinstance(retry, AsyncRetryPolicy) and retry.total_retries == 3 assert isinstance(redirect, AsyncRedirectPolicy) and redirect.allow is False - @pytest.mark.parametrize("status", _model_runners._DECISION_RETRY_STATUS_CODES) + # Statuses that azure-core's retry policy retries by default and a Decision endpoint can return. + @pytest.mark.parametrize("status", [408, 429, 500, 502, 503, 504, 529]) def test_transient_post_response_is_retried_without_retry_after(self, monkeypatch, status): from azure.ai.evaluation._http_utils import AsyncHttpPipeline @@ -451,6 +455,113 @@ def test_provider_http_error_code_preserves_classification_and_accepted_metadata else {"decision_model": {"deployment": "Decision-1"}} ) + @pytest.mark.parametrize( + "detail,kind", + [ + ( + "invalid model request: Question prompt exceeds the 32768-token context limit including the decision " + "token", + "context_window_exceeded", + ), + ( + "invalid TypeSafe request: request is about 75108 tokens; the limit for the state plus all questions " + "is 64000", + "request_token_limit_exceeded", + ), + ("invalid TypeSafe request: request has 256 questions; the limit is 255", "question_limit_exceeded"), + ( + 'invalid TypeSafe request: invalid instructions for question "private-key": value must not be empty', + "invalid_question", + ), + ( + 'invalid TypeSafe request: invalid criteria for question "q": value must not be empty', + "invalid_question", + ), + ("invalid TypeSafe request: private-message test-key-value", None), + ( + "invalid model request: Question prompt exceeds the 32768-token context limit " + "x" * 5000, + "context_window_exceeded", + ), + ( + 'invalid TypeSafe request: invalid instructions for question "' + + "q" * 2000 + + '": value must not be empty', + "invalid_question", + ), + ("x" * 1100 + "invalid model request: Question prompt exceeds the 32768-token context limit", None), + ("", None), + ], + ) + def test_provider_rejection_reason_is_named_by_a_known_kind_without_its_text(self, client, detail, kind): + client.response = FakeResponse(422, json.dumps({"detail": detail}).encode()) + with pytest.raises(EvaluationException) as raised: + run(DecisionModelRunner(DECISION_CONFIG, max_retries=0)) + diagnostic = f" ({kind})" if kind else "" + assert str(raised.value) == f"(UserError) The decision model returned HTTP 422{diagnostic}." + assert raised.value.blame == ErrorBlame.USER_ERROR + native = raised.value.properties["decision_model"] + assert native.get("error_kind") == kind + persisted = json.dumps(raised.value.properties) + str(raised.value) + raised.value.internal_message + assert "private" not in persisted and "test-key-value" not in persisted and "32768" not in persisted + assert len(client.calls) == 1 + + @pytest.mark.parametrize( + "body", + [ + {"detail": ["exceeds the 1-token context limit"]}, + {"detail": {"message": "exceeds the 1-token context limit"}}, + {"error": "exceeds the 1-token context limit"}, + ["exceeds the 1-token context limit"], + ], + ) + def test_malformed_provider_rejection_detail_has_no_kind(self, client, body): + client.response = FakeResponse(422, json.dumps(body).encode()) + with pytest.raises(EvaluationException) as raised: + run(DecisionModelRunner(DECISION_CONFIG, max_retries=0)) + assert str(raised.value) == "(UserError) The decision model returned HTTP 422." + assert "error_kind" not in raised.value.properties["decision_model"] + + def test_provider_error_code_names_the_message_and_only_a_422_has_a_kind(self, client): + body = { + "error": {"code": "unsupported_request_argument", "message": "private-message"}, + "detail": "invalid model request: Question prompt exceeds the 32768-token context limit", + } + client.response = FakeResponse(400, json.dumps(body).encode()) + with pytest.raises(EvaluationException) as raised: + run(DecisionModelRunner(DECISION_CONFIG, max_retries=0)) + assert str(raised.value) == "(UserError) The decision model returned HTTP 400 (unsupported_request_argument)." + assert "error_kind" not in raised.value.properties["decision_model"] + assert "private-message" not in json.dumps(raised.value.properties) + + @pytest.mark.parametrize( + "detail", + [ + 'invalid TypeSafe request: invalid instructions for question "exceeds the 32768-token context limit": ' + "value must not be empty", + 'invalid TypeSafe request: invalid criteria for question "request has 300 questions; the limit is 255": ' + "value must not be empty", + ], + ) + def test_a_quoted_question_identifier_cannot_change_the_kind(self, client, detail): + client.response = FakeResponse(422, json.dumps({"detail": detail}).encode()) + with pytest.raises(EvaluationException) as raised: + run(DecisionModelRunner(DECISION_CONFIG, max_retries=0)) + assert raised.value.properties["decision_model"]["error_kind"] == "invalid_question" + + @pytest.mark.parametrize( + "limits", + [{"max_request_bytes": 64}, {"max_state_question_bytes": 64}], + ids=["request", "state_question"], + ) + def test_request_over_the_byte_limits_names_its_reason_and_is_not_sent(self, client, limits): + with pytest.raises(EvaluationException, match="exceeds the configured byte budget") as raised: + run(DecisionModelRunner(DECISION_CONFIG, **limits), state={"response": "x" * 200}) + assert raised.value.blame == ErrorBlame.USER_ERROR + native = raised.value.properties["decision_model"] + assert native["error_kind"] == "request_bytes_exceeded" + assert native["execution"]["attempt_count"] == 0 + assert not client.calls + @pytest.mark.parametrize( "payload", [ @@ -2042,3 +2153,102 @@ def test_optional_metrics_absence_does_not_change_summary_or_require_a_logger(se with patch.dict(sys.modules, {"opentelemetry.metrics": None}): summary = execution.finish("error") assert summary["attempt_count"] == 0 and summary["usage_state"] == "unknown" + + +DECLARED_ORDER = ["response", "context", "query", "tool_definitions"] + + +class PassThroughDecisionEvaluator(ModelEvaluatorBase): + """Sends its converted inputs to the decision model unchanged, as the catalog's task completion evaluator does.""" + + _supports_decision_models = True + + def _derive_singleton_inputs(self): + return [["response", "context", "query"], ["query", "response", "tool_definitions"]] + + async def _do_eval(self, eval_input): + return await DecisionEvaluationStrategy( + self._decision_model_runner, + result_key="probe", + error_target=ErrorTarget.EVALUATE, + questions={ + "status": {"type": "noul", "instructions": "Applicable?"}, + "probe": {"type": "noul", "instructions": "Good?"}, + }, + scoring=BinaryScoring(0.5), + threshold=1, + timeout=self._LLM_CALL_TIMEOUT, + ).evaluate(eval_input) + + +# Runs in a fresh interpreter, because the hash seed is fixed when the interpreter starts. +HASH_SEED_PROBE = """ +import json +import azure.ai.evaluation +from azure.ai.evaluation._evaluators._common import EvaluatorBase + + +class Probe(EvaluatorBase): + def _derive_singleton_inputs(self): + return [["response", "context", "query"], ["query", "response", "tool_definitions", "ground_truth"]] + + async def _do_eval(self, eval_input): + return {} + + +eval_input = Probe()._convert_kwargs_to_eval_input(query="q", response="r", context="c") +print(json.dumps({"file": azure.ai.evaluation.__file__, "order": list(eval_input[0])})) +""" + + +class TestDecisionStateOrder: + def test_inputs_keep_the_order_their_overloads_declare(self): + evaluator = PassThroughDecisionEvaluator(result_key="probe", model_config=DECISION_CONFIG) + assert evaluator._get_all_singleton_inputs() == DECLARED_ORDER + eval_input = evaluator._convert_kwargs_to_eval_input(query="q", response="r", tool_definitions=[{"name": "t"}]) + assert list(eval_input[0]) == DECLARED_ORDER + + def test_a_built_in_with_several_overloads_keeps_its_declared_order(self): + assert GroundednessEvaluator(CHAT_CONFIG)._get_all_singleton_inputs() == DECLARED_ORDER + + def test_a_decision_request_sends_unchanged_inputs_in_their_declared_order(self, client): + payload = decision_payload(0.9, probe={"type": "noul", "noul": 0.9}) + client.response = FakeResponse(body=json.dumps(payload).encode()) + evaluator = PassThroughDecisionEvaluator(result_key="probe", model_config=DECISION_CONFIG) + evaluator(query="q", response="r", tool_definitions=[{"name": "t"}]) + assert client.calls[0]["content"].startswith( + b'{"model":"Decision-1","state":{"response":"r","context":null,"query":"q","tool_definitions":[{"name":"t"}]},' + ) + + def test_the_order_is_the_same_for_every_hash_seed(self, tmp_path): + import azure.ai.evaluation + + # The children import the same package as this process, from a checkout or an installed wheel. They start in + # an empty directory, because "python -c" puts the working directory first on the path, so a checkout there + # would shadow an installed wheel. + loaded = Path(azure.ai.evaluation.__file__).resolve() + search_path = os.pathsep.join(filter(None, [str(loaded.parents[3]), os.environ.get("PYTHONPATH")])) + children = [] + try: + for seed in range(1, 4): + environment = dict(os.environ, PYTHONHASHSEED=str(seed), PYTHONNOUSERSITE="1", PYTHONPATH=search_path) + children.append( + subprocess.Popen( + [sys.executable, "-c", HASH_SEED_PROBE], + cwd=tmp_path, + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + ) + for seed, child in enumerate(children, start=1): + output, errors = child.communicate(timeout=180) + assert child.returncode == 0, errors[-2000:] + result = json.loads(output.strip().splitlines()[-1]) + assert Path(result["file"]).resolve() == loaded + assert result["order"] == DECLARED_ORDER + ["ground_truth"], f"PYTHONHASHSEED={seed}" + finally: + for child in children: + if child.poll() is None: + child.kill() diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index 79ed493819aa..c26485ff1bd2 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -782,6 +782,43 @@ def test_binary_aggregation_skips_unhashable_result_columns(self, caplog): f"got: {[r.getMessage() for r in caplog.records]}" ) + def test_binary_aggregation_counts_only_rows_that_passed_or_failed(self): + data = { + "outputs.mixed.mixed_result": ["pass", "fail", "not_applicable", None, "pass", np.nan], + "outputs.mixed.mixed": [1.0, 0.0, None, np.nan, 1.0, np.nan], + "outputs.skipped.skipped_result": ["not_applicable", None, "not_applicable", np.nan, None, np.nan], + "outputs.skipped.skipped": [None, np.nan, None, np.nan, None, np.nan], + "outputs.passing.passing_result": ["pass"] * 6, + "outputs.failing.failing_result": ["fail", None, "fail", "not_applicable", np.nan, "fail"], + "outputs.custom.custom_result": [True, False, True, True, False, True], + } + + aggregation = _aggregate_metrics(pd.DataFrame(data), {}) + + assert aggregation["mixed.binary_aggregate"] == 0.67 + assert aggregation["mixed.mixed"] == pytest.approx(2 / 3) + assert aggregation["passing.binary_aggregate"] == 1.0 + assert aggregation["failing.binary_aggregate"] == 0.0 + # Like the mean score, the binary aggregate is left out when no row has a result. + assert "skipped.binary_aggregate" not in aggregation + assert "skipped.skipped" not in aggregation + # Only "pass" and "fail" are results, so a column of other values has no binary aggregate. + assert "custom.binary_aggregate" not in aggregation + + def test_binary_aggregation_reports_the_last_result_column_of_an_evaluator(self): + data = { + "outputs.qa.f1_score_result": ["pass", "pass", "fail"], + "outputs.qa.groundedness_result": ["not_applicable", None, "not_applicable"], + "outputs.rouge.rouge_precision_result": ["not_applicable", None, "not_applicable"], + "outputs.rouge.rouge_recall_result": ["pass", "fail", None], + } + + aggregation = _aggregate_metrics(pd.DataFrame(data), {}) + + # An earlier column's rate does not stand in for a last column without results. + assert "qa.binary_aggregate" not in aggregation + assert aggregation["rouge.binary_aggregate"] == 0.5 + def test_aggregate_label_defect_metrics_with_nan_in_details(self): """Test that NaN/None values in details column are properly ignored during aggregation.""" data = { diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py new file mode 100644 index 000000000000..82ce227c8ad9 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py @@ -0,0 +1,506 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +"""Tool Call Success reads the judge's score where judge models put it, and skips responses without tool calls.""" + +import json +import math +import re +from pathlib import Path +from unittest.mock import AsyncMock + +import pandas as pd +import pytest + +from azure.ai.evaluation import evaluate +from azure.ai.evaluation._evaluators._tool_call_success import _tool_call_success as module +from azure.ai.evaluation._evaluators._tool_call_success._tool_call_success import ( + _ToolCallSuccessEvaluator, + _has_no_tool_calls, +) +from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException + +CHAT_CONFIG = { + "azure_endpoint": "https://example.openai.azure.com", + "azure_deployment": "judge", + "api_key": "test-key", +} +DECISION_CONFIG = { + "type": "decision", + "azure_endpoint": "https://example.services.ai.azure.com", + "azure_deployment": "Decision-1", + "api_key": "test-key", +} +PROMPTY = Path(module.__file__).parent / "tool_call_success.prompty" +TOOL_CALL_RESPONSE = [ + { + "role": "assistant", + "content": [ + {"type": "tool_call", "tool_call_id": "call_1", "name": "fetch_weather", "arguments": {"city": "Paris"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": [{"type": "tool_result", "tool_result": {"sky": "sunny"}}]}, + {"role": "assistant", "content": [{"type": "text", "text": "It is sunny in Paris."}]}, +] +TEXT_RESPONSE = [ + {"role": "user", "content": [{"type": "text", "text": "What is the weather in Paris?"}]}, + {"role": "assistant", "content": [{"type": "output_text", "text": "I can't check the weather."}]}, +] +# What OpenAI's ChatCompletionMessage.model_dump() returns for a reply without tool calls. +OPENAI_TEXT_REPLY = { + "content": "Hello.", + "refusal": None, + "role": "assistant", + "annotations": [], + "audio": None, + "function_call": None, + "tool_calls": None, +} +OPENAI_TOOL_CALL_RESPONSE = [ + { + **OPENAI_TEXT_REPLY, + "content": "Checking the weather.", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "fetch_weather", "arguments": '{"city": "Paris"}'}, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [{"type": "tool_result", "tool_result": {"error": "timed out"}}], + }, + {**OPENAI_TEXT_REPLY, "content": "The weather service timed out."}, +] +JUDGE_USAGE = {"input_token_count": 11, "output_token_count": 7, "total_token_count": 18} + + +def judge(llm_output): + """Return a chat evaluator whose judge model answers with ``llm_output``.""" + evaluator = _ToolCallSuccessEvaluator(model_config=dict(CHAT_CONFIG)) + evaluator._flow = AsyncMock(return_value={"llm_output": llm_output, **JUDGE_USAGE}) + return evaluator + + +def evaluate_with(llm_output): + return judge(llm_output)(response=TOOL_CALL_RESPONSE) + + +def prompty_example_outputs(): + text = PROMPTY.read_text(encoding="utf-8") + decoder = json.JSONDecoder() + return [ + decoder.raw_decode(text, text.index("{", match.end()))[0] for match in re.finditer(r"EXPECTED OUTPUT", text) + ] + + +@pytest.mark.unittest +class TestToolCallSuccessPrompty: + def test_every_example_output_is_valid_json_with_a_top_level_score(self): + outputs = prompty_example_outputs() + + assert len(outputs) == 14 + for output in outputs: + assert set(output) == {"reason", "properties", "score", "status"} + assert output["status"] == "completed" and output["score"] in (0, 1) + assert set(output["properties"]) == {"failed_tools"} + + +@pytest.mark.unittest +class TestToolCallSuccessScore: + @pytest.mark.parametrize( + "llm_output, expected", + [ + ({"score": 1, "status": "completed", "properties": {"failed_tools": ""}}, 1.0), + ({"score": 0, "status": "completed", "properties": {"failed_tools": "fetch_weather"}}, 0.0), + # Judge models such as gpt-4.1 and gpt-4o nested the score and status in properties. + ({"properties": {"failed_tools": "", "score": 1, "status": "completed"}}, 1.0), + ({"properties": {"failed_tools": "fetch_weather", "score": 0, "status": "completed"}}, 0.0), + # A top-level value wins over a nested one. + ({"score": 0, "properties": {"score": 1}}, 0.0), + ({"score": 1, "properties": {"score": 0}}, 1.0), + ({"score": None, "properties": {"score": 1}}, 1.0), + # Values that were already read as 0 or 1 still are. + ({"score": 1.0}, 1.0), + ({"score": "1"}, 1.0), + ({"score": "0"}, 0.0), + ({"score": True}, 1.0), + ({"score": False}, 0.0), + ], + ) + def test_the_score_is_read_from_the_top_level_or_from_properties(self, llm_output, expected): + result = evaluate_with({"reason": "Recorded reason.", **llm_output}) + + assert result["tool_call_success"] == expected + assert result["tool_call_success_score"] == expected + assert result["tool_call_success_result"] == ("pass" if expected == 1.0 else "fail") + assert result["tool_call_success_passed"] is (expected == 1.0) + assert result["tool_call_success_status"] == "completed" + assert result["tool_call_success_reason"] == "Recorded reason." + assert result["tool_call_success_properties"]["prompt_tokens"] == 11 + + def test_nested_keys_stay_in_the_returned_properties(self): + result = evaluate_with({"reason": "r", "properties": {"failed_tools": "", "score": 1, "status": "completed"}}) + + assert {"failed_tools": "", "score": 1, "status": "completed"}.items() <= ( + result["tool_call_success_properties"].items() + ) + + @pytest.mark.parametrize( + "llm_output", + [ + {"status": "skipped", "reason": "No tool calls."}, + {"properties": {"status": "skipped"}, "reason": "No tool calls."}, + {"properties": {"status": "skipped", "reason": "No tool calls."}}, + ], + ) + def test_a_skipped_status_is_not_applicable(self, llm_output): + result = evaluate_with(llm_output) + + assert result["tool_call_success_result"] == "not_applicable" + assert result["tool_call_success_reason"] == "Not applicable: No tool calls." + assert result["tool_call_success_status"] == "skipped" + assert result["tool_call_success"] is None + + def test_a_top_level_status_wins_over_a_nested_skip(self): + result = evaluate_with({"status": "completed", "score": 1, "properties": {"status": "skipped"}}) + + assert result["tool_call_success_status"] == "completed" + assert result["tool_call_success_result"] == "pass" + + @pytest.mark.parametrize( + "llm_output, shown", + [ + ({"reason": "No score anywhere.", "status": "completed"}, "None"), + ({"score": None, "properties": None}, "None"), + ({"score": None, "properties": {"failed_tools": ""}}, "None"), + ({"score": 0.5}, "0.5"), + ({"score": 2}, "2"), + ({"score": -1}, "-1"), + ({"score": math.nan}, "nan"), + ({"score": math.inf}, "inf"), + ({"score": "pass"}, "'pass'"), + ({"score": [1]}, "[1]"), + ({"score": {"value": 1}}, "{'value': 1}"), + ({"properties": {"score": 0.5}}, "0.5"), + ({"score": 0.5, "properties": {"score": 1}}, "0.5"), + ({"properties": "score: 1"}, "None"), + ], + ) + def test_an_output_without_a_score_of_0_or_1_is_an_error_not_a_failed_tool_call(self, llm_output, shown): + with pytest.raises(EvaluationException) as raised: + evaluate_with(llm_output) + + assert str(raised.value) == f"(InternalError) Invalid score value: {shown}. Expected 0 or 1." + assert raised.value.internal_message == "Invalid score value." + assert raised.value.blame == ErrorBlame.SYSTEM_ERROR + assert raised.value.category == ErrorCategory.FAILED_EXECUTION + assert raised.value.target == ErrorTarget.TOOL_CALL_SUCCESS_EVALUATOR + + def test_an_invalid_score_is_shown_shortened(self): + with pytest.raises(EvaluationException) as raised: + evaluate_with({"score": "x" * 500}) + + assert len(str(raised.value)) < 100 + + def test_evaluate_reports_a_row_without_a_score_as_an_error(self, tmp_path): + rows = [ + {"response": TOOL_CALL_RESPONSE, "case": "valid"}, + {"response": TOOL_CALL_RESPONSE, "case": "missing"}, + {"response": TOOL_CALL_RESPONSE, "case": "nested"}, + ] + data = tmp_path / "rows.jsonl" + data.write_text("\n".join(json.dumps(row) for row in rows), encoding="utf-8") + outputs = iter( + [ + {"score": 1, "status": "completed", "reason": "valid"}, + {"status": "completed", "reason": "missing"}, + {"properties": {"score": 1, "status": "completed"}, "reason": "nested"}, + ] + ) + evaluator = _ToolCallSuccessEvaluator(model_config=dict(CHAT_CONFIG)) + + async def flow(timeout, **kwargs): + return {"llm_output": next(outputs), **JUDGE_USAGE} + + evaluator._flow = flow + result = evaluate( + data=str(data), + evaluators={"tool_call_success": evaluator}, + evaluator_config={"tool_call_success": {"column_mapping": {"response": "${data.response}"}}}, + _use_run_submitter_client=True, + ) + reason, score = ( + "outputs.tool_call_success.tool_call_success_reason", + "outputs.tool_call_success.tool_call_success", + ) + completed = {row[reason]: row[score] for row in result["rows"] if isinstance(row.get(reason), str)} + errored = [row for row in result["rows"] if not isinstance(row.get(reason), str)] + + assert completed == {"valid": 1.0, "nested": 1.0} + assert len(errored) == 1 and pd.isna(errored[0][score]) + + +@pytest.mark.unittest +class TestToolCallSuccessWithoutToolCalls: + @pytest.mark.parametrize("config", [CHAT_CONFIG, DECISION_CONFIG], ids=["chat", "decision"]) + @pytest.mark.parametrize( + "response", + [ + TEXT_RESPONSE, + [{"role": "assistant", "content": "I can't check the weather."}], + [ + {"role": "system", "content": "Answer briefly."}, + {"role": "developer", "content": [{"type": "input_text", "text": "Be polite."}]}, + {"role": "assistant", "content": [{"type": "text", "text": "Hello."}]}, + ], + [ + { + "role": "assistant", + "content": [{"type": "mcp_approval_request", "name": "search", "arguments": "{}"}], + }, + { + "role": "tool", + "tool_call_id": "approval_1", + "content": [{"type": "mcp_approval_response", "approve": False}], + }, + ], + [{"role": "assistant", "content": "Hello.", "tool_calls": None}], + [{"role": "assistant", "content": [{"type": "text", "text": "Hello."}], "tool_calls": []}], + [{"role": "assistant", "content": "Hello.", "function_call": None}], + [{"role": "user", "content": "Hi.", "tool_call_id": None}, {"role": "assistant", "content": "Hello."}], + [OPENAI_TEXT_REPLY], + ], + ids=[ + "text-items", + "string-content", + "every-text-role", + "mcp-approval-only", + "null-tool-calls", + "empty-tool-calls", + "null-function-call", + "null-tool-call-id", + "openai-model-dump", + ], + ) + def test_a_response_without_tool_calls_is_not_applicable_without_a_judge_call(self, config, response): + evaluator = _ToolCallSuccessEvaluator(model_config=dict(config)) + if evaluator._decision is None: + evaluator._flow = AsyncMock(side_effect=AssertionError("The judge was called.")) + else: + evaluator._decision.evaluate = AsyncMock(side_effect=AssertionError("The judge was called.")) + + result = evaluator(response=response) + + assert result == { + "tool_call_success": None, + "tool_call_success_score": None, + "tool_call_success_passed": None, + "tool_call_success_result": "not_applicable", + "tool_call_success_reason": "Not applicable: No tool calls found in response.", + "tool_call_success_status": "skipped", + "tool_call_success_threshold": 1, + "tool_call_success_properties": None, + } + + @pytest.mark.parametrize( + "response", + [ + TOOL_CALL_RESPONSE, + [ + { + "role": "assistant", + "content": [ + { + "type": "function_call", + "tool_call_id": "call_1", + "name": "fetch_weather", + "arguments": {"city": "Paris"}, + }, + ], + }, + {"role": "assistant", "content": [{"type": "text", "text": "Done."}]}, + ], + "[TOOL_CALL] fetch_weather(city=Paris)", + "It is sunny in Paris.", + [ + { + "role": "assistant", + "content": '[TOOL_CALL] fetch_weather(city="Paris")\n[TOOL_RESULT] {"error": "timed out"}', + } + ], + [{"role": "assistant", "content": [{"type": "text", "text": '[TOOL_RESULT] {"error": "timed out"}'}]}], + [ + { + "role": "assistant", + "content": [ + {"type": "mcp_approval_request", "name": "search", "arguments": "{}"}, + { + "type": "tool_call", + "tool_call_id": "call_1", + "name": "fetch_weather", + "arguments": {"city": "Paris"}, + }, + ], + }, + {"role": "assistant", "content": [{"type": "text", "text": "Done."}]}, + ], + OPENAI_TOOL_CALL_RESPONSE, + ], + ids=[ + "tool-call", + "function-call", + "tool-call-text", + "plain-text", + "trace-in-message", + "trace-in-text-item", + "approval-with-tool-call", + "openai-model-dump-tool-call", + ], + ) + def test_tool_calls_and_text_responses_still_go_to_the_judge(self, response): + evaluator = judge({"score": 1, "status": "completed", "reason": "r"}) + + result = evaluator(response=response) + + assert evaluator._flow.await_count == 1 + assert result["tool_call_success_result"] == "pass" + + def test_a_decision_judge_still_gets_tool_calls(self): + evaluator = _ToolCallSuccessEvaluator(model_config=dict(DECISION_CONFIG)) + evaluator._decision.evaluate = AsyncMock(return_value={"tool_call_success": 1.0}) + + assert evaluator(response=TOOL_CALL_RESPONSE)["tool_call_success"] == 1.0 + state = evaluator._decision.evaluate.await_args.args[0] + assert "[TOOL_CALL] fetch_weather" in state["tool_calls"] + + def test_a_decision_judge_still_gets_openai_tool_calls(self): + evaluator = _ToolCallSuccessEvaluator(model_config=dict(DECISION_CONFIG)) + evaluator._decision.evaluate = AsyncMock(return_value={"tool_call_success": 0.0}) + + assert evaluator(response=OPENAI_TOOL_CALL_RESPONSE)["tool_call_success"] == 0.0 + assert evaluator._decision.evaluate.await_count == 1 + assert "fetch_weather" in str(evaluator._decision.evaluate.await_args.args[0]["tool_calls"]) + + def test_evaluate_leaves_rows_without_tool_calls_out_of_the_pass_rate(self, tmp_path): + failed_call = json.loads(json.dumps(TOOL_CALL_RESPONSE).replace("sunny", "timed out")) + rows = [{"response": TOOL_CALL_RESPONSE}, {"response": failed_call}, {"response": [OPENAI_TEXT_REPLY]}] + data = tmp_path / "rows.jsonl" + data.write_text("\n".join(json.dumps(row) for row in rows), encoding="utf-8") + evaluator = _ToolCallSuccessEvaluator(model_config=dict(CHAT_CONFIG)) + + async def flow(timeout, **kwargs): + score = 0 if "timed out" in str(kwargs["tool_calls"]) else 1 + return {"llm_output": {"score": score, "status": "completed", "reason": "r"}, **JUDGE_USAGE} + + evaluator._flow = flow + result = evaluate( + data=str(data), + evaluators={"tool_call_success": evaluator}, + evaluator_config={"tool_call_success": {"column_mapping": {"response": "${data.response}"}}}, + _use_run_submitter_client=True, + ) + + results = sorted(row["outputs.tool_call_success.tool_call_success_result"] for row in result["rows"]) + assert results == ["fail", "not_applicable", "pass"] + assert result["metrics"]["tool_call_success.tool_call_success"] == 0.5 + assert result["metrics"]["tool_call_success.binary_aggregate"] == 0.5 + + @pytest.mark.parametrize( + "messages", + [ + [], + "text", + [{"role": "tool", "tool_call_id": "call_1", "content": [{"type": "tool_result", "tool_result": "ok"}]}], + [{"role": "assistant", "content": [{"type": "tool_call", "name": "t", "arguments": {}}]}], + [{"role": "assistant", "content": "Calling a tool.", "tool_calls": [{"id": "call_1"}]}], + [{"role": "assistant", "content": "Calling a tool.", "function_call": {"name": "t"}}], + [ + { + "role": "assistant", + "content": "Calling a tool.", + "tool_calls": [{"id": "call_1"}], + "function_call": None, + } + ], + [{"role": "assistant", "content": "Calling a tool.", "tool_calls": [None]}], + [{"role": "assistant", "content": "Calling a tool.", "function_call": "t"}], + [{"role": "assistant", "content": "Calling a tool.", "tool_calls": False}], + [{"role": "assistant", "content": "Result.", "tool_call_id": "call_1"}], + [{"role": "function", "content": "Result."}], + [{"content": "No role."}], + [{"role": "assistant", "content": None}], + [{"role": "assistant", "content": [{"type": "mcp_call", "name": "t"}]}], + [{"role": "assistant", "content": [{"type": "image_url", "image_url": "https://example.com"}]}], + [{"role": "assistant", "content": ["text"]}], + ["text"], + [{"role": "assistant", "content": "[TOOL_CALL] fetch_weather()"}], + [{"role": "assistant", "content": [{"type": "output_text", "text": "[TOOL_RESULT] ok"}]}], + [{"role": "assistant", "content": [{"type": "text", "text": None}]}], + [{"role": ["assistant"], "content": "Hello."}], + [{"role": {"name": "assistant"}, "content": "Hello."}], + [{"role": "assistant", "content": [{"type": {"kind": "text"}, "text": "Hello."}]}], + [{"role": "assistant", "content": [{"type": ["text"], "text": "Hello."}]}], + [{"role": "assistant", "content": [{"type": {"kind": "mcp_approval_request"}}]}], + ], + ) + def test_only_recognized_text_messages_count_as_having_no_tool_calls(self, messages): + assert _has_no_tool_calls(messages) is False + + @pytest.mark.parametrize("role", [["assistant"], {"name": "assistant"}], ids=["list", "dict"]) + def test_an_unsupported_role_value_still_goes_to_the_judge(self, role): + evaluator = judge({"score": 1, "status": "completed", "reason": "r"}) + + result = evaluator(response=[{"role": role, "content": "Hello."}]) + + assert evaluator._flow.await_count == 1 + assert result["tool_call_success_result"] == "pass" + + @pytest.mark.parametrize("item_type", [{"kind": "text"}, ["text"]], ids=["dict", "list"]) + def test_an_unsupported_content_type_value_gets_the_input_validation_error(self, item_type): + evaluator = judge({"score": 1, "status": "completed", "reason": "r"}) + + with pytest.raises(EvaluationException, match="Invalid content type") as raised: + evaluator(response=[{"role": "assistant", "content": [{"type": item_type, "text": "Hello."}]}]) + + assert raised.value.blame == ErrorBlame.USER_ERROR + assert evaluator._flow.await_count == 0 + + def test_text_messages_have_no_tool_calls(self): + assert _has_no_tool_calls(TEXT_RESPONSE) is True + assert _has_no_tool_calls([{"role": "assistant", "content": []}]) is True + + @pytest.mark.parametrize("key", ["tool_calls", "function_call", "tool_call_id"]) + @pytest.mark.parametrize("value", [None, "", [], {}], ids=["none", "empty-string", "empty-list", "empty-dict"]) + def test_a_call_field_that_holds_nothing_counts_as_absent(self, key, value): + assert _has_no_tool_calls([{"role": "assistant", "content": "Hello.", key: value}]) is True + + def test_mcp_approvals_are_not_tool_calls_but_a_tool_call_beside_one_is(self): + approval = {"type": "mcp_approval_request", "name": "search", "arguments": "{}"} + response = {"type": "mcp_approval_response", "approve": True} + text = {"type": "text", "text": "Searching."} + tool_call = {"type": "tool_call", "tool_call_id": "call_1", "name": "search", "arguments": {}} + + assert _has_no_tool_calls( + [ + {"role": "assistant", "content": [approval, text]}, + {"role": "tool", "tool_call_id": "approval_1", "content": [response]}, + ] + ) + assert not _has_no_tool_calls([{"role": "assistant", "content": [approval, tool_call]}]) + assert not _has_no_tool_calls( + [{"role": "tool", "tool_call_id": "call_1", "content": [response, {"type": "tool_result"}]}] + ) + # Tool calls in message fields count even beside approval-only content. + assert not _has_no_tool_calls( + [ + {"role": "assistant", "content": [approval], "tool_calls": [{"id": "call_1", "type": "function"}]}, + {"role": "assistant", "content": [text]}, + ] + ) + assert not _has_no_tool_calls([{"role": "assistant", "content": [approval], "function_call": {"name": "t"}}]) + assert _has_no_tool_calls([{"role": "assistant", "content": [approval], "tool_calls": None}])