From 8dc6b355a7512bd857281bd455b954577c062441 Mon Sep 17 00:00:00 2001 From: Ahmad Qardahji Date: Tue, 3 Nov 2026 20:45:41 -0800 Subject: [PATCH 1/5] Send free-text tool transcripts in Tool Call Success message lists to the judge Since #49804, a list of text messages was reported as not applicable without a judge call unless its text held this evaluator's [TOOL_CALL] or [TOOL_RESULT] markers. A transcript in another text format, such as ReAct Action and Observation lines or tags, was left out of the pass rate, together with its failed tool calls. Text that contains tool_call, tool_result, tool_response, function_call or function_result in any letter case, Llama's <|python_tag|> or token, or a line that starts with a label such as Action:, Observation:, Tool call: or Tool result: now counts as a tool transcript. Such a list goes to the judge model as its text, with each message's role and every tool definition, as a string does, because Decision-1 answered not applicable for ReAct steps inside a list of messages but judged the same text correctly. Also corrects the 1.18.11 CHANGELOG entries for error_kind, the score check and binary_aggregate. --- .../azure-ai-evaluation/CHANGELOG.md | 8 +- .../_tool_call_success/_tool_call_success.py | 91 +++++++-- .../test_tool_call_success_evaluator.py | 182 +++++++++++++++++- 3 files changed, 254 insertions(+), 27 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index 4f7775588b05..b3fe3c4a940e 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -8,13 +8,13 @@ - `QAEvaluator` accepts a Decision model configuration and passes `credential` and the Decision execution controls to its model-based evaluators, which run on separate threads, so the credential must be synchronous. With a chat model they are not passed, as before. - The binary evaluators (task adherence, task completion, tool call success, tool input accuracy, tool output utilization and tool selection) accept a `probability_threshold` keyword. For Decision models it sets the probability at or above which the score is 1 (default 0.75) and overrides the value in `model_config`; chat models ignore it with a warning. - Conversation inputs are handled as before with a Decision model: evaluators that accept them score each user and assistant turn separately and aggregate the results. The catalog's whole-conversation rubrics are not included. -- A failed Decision request names a recognized reason in `decision_model.error_kind` of the error's properties, which is kept on failed result rows, and in the error message, for example `The decision model returned HTTP 422 (context_window_exceeded).` The provider states these reasons for an HTTP 422 only in text, which is never exposed: `context_window_exceeded` when the evidence and one question exceed the model's context window, `request_token_limit_exceeded` when the evidence and all questions exceed the model's request limit, `question_limit_exceeded` and `invalid_question`. A request over `max_request_bytes` or `max_state_question_bytes`, which is not sent, has the kind `request_bytes_exceeded`. +- A failed Decision request names a recognized reason in `decision_model.error_kind` of the error's properties, which is kept on failed result rows. The error message also names it when the provider returns no error code, for example `The decision model returned HTTP 422 (context_window_exceeded).`, so read the property rather than the message. The provider states these reasons for an HTTP 422 only in text, which is never exposed: `context_window_exceeded` when the evidence and one question exceed the model's context window, `request_token_limit_exceeded` when the evidence and all questions exceed the model's request limit, `question_limit_exceeded` and `invalid_question`. A request over `max_request_bytes` or `max_state_question_bytes`, which is not sent, has the kind `request_bytes_exceeded`, and its error message is unchanged. ### Bugs Fixed -- `_ToolCallSuccessEvaluator` no longer scores successful tool calls 0 when a chat judge model, such as gpt-4.1 or gpt-4o, puts `score` and `status` inside `properties`. The example outputs in its prompt are now valid JSON, and `score`, `status` and `reason` are read from `properties` when the output has none at the top level. A missing score is no longer reported as 0, and any score other than 0 or 1 raises a system-classified `EvaluationException`, as `_ToolInputAccuracyEvaluator` does. -- `_ToolCallSuccessEvaluator` reports a list of messages that holds no tool calls, such as one that contains only text or only MCP approval messages, as not applicable without calling the chat or Decision judge model. A `tool_calls`, `function_call` or `tool_call_id` field that is None or empty, as in OpenAI's `model_dump()` of a reply without tool calls, does not count as a tool call. Strings, text that contains `[TOOL_CALL]` or `[TOOL_RESULT]`, and messages with any other content are still sent to the judge model. -- `evaluate()` computes `.binary_aggregate` over the rows whose result is `pass` or `fail`. Other rows, such as not applicable or errored rows, used to count as failures, although the mean score leaves them out. An evaluator without `pass` or `fail` rows, including a custom evaluator whose results have other values, has no `binary_aggregate`, as one without scores has no mean. +- `_ToolCallSuccessEvaluator` no longer scores successful tool calls 0 when a chat judge model, such as gpt-4.1 or gpt-4o, puts `score` and `status` inside `properties`. The example outputs in its prompt are now valid JSON, and `score`, `status` and `reason` are read from `properties` when the output has none at the top level. A missing score is no longer reported as 0, and a score that does not read as 0 or 1 raises a system-classified `EvaluationException`. Values that read as 0 or 1, such as `True`, `"1"` and `1.0`, are still accepted. +- `_ToolCallSuccessEvaluator` reports a list of messages that holds no tool calls, such as one that contains only text or only MCP approval messages, as not applicable without calling the chat or Decision judge model. A `tool_calls`, `function_call` or `tool_call_id` field that is None or empty, as in OpenAI's `model_dump()` of a reply without tool calls, does not count as a tool call. Strings, messages with any other content, and text that looks like a tool transcript are still sent to the judge model. Such text contains `tool_call`, `tool_result`, `tool_response`, `function_call` or `function_result` in any letter case, as tags such as ``, `[TOOL_CALLS]` and this evaluator's `[TOOL_RESULT]` do, Llama's `<|python_tag|>` or `` token, or has a line that starts with a label such as `Action:`, `Observation:`, `Tool call:` or `Tool result:`, as ReAct agents write. A list of text messages with such text is sent as its text, with each message's role and every tool definition, as a string is, because a Decision model can miss ReAct steps inside a list of messages. Pass a transcript in any other text format as a string for the judge model to assess it. +- `evaluate()` computes `.binary_aggregate` over the rows whose result is `pass` or `fail`. Other rows, such as not applicable or errored rows, used to count as failures, although the mean score leaves them out. An evaluator without `pass` or `fail` rows, including a custom evaluator whose results have other values, has no `binary_aggregate`, as one without scores has no mean. With several result columns, the last one whose values can be counted decides, as before, so an evaluator whose `pass` and `fail` rows are only in an earlier column also has no `binary_aggregate`. - An evaluator's inputs keep the order in which its `__call__` overloads declare them, instead of the order of a Python `set`, which varies with the process's hash seed. An evaluator that sends these inputs to a Decision model unchanged, as several Microsoft Foundry catalog evaluators do, could list the same fields in a different order in each process, and the Decision model could answer the same input differently. The built-in evaluators in this package already send their Decision inputs in a fixed order. ## 1.18.10 (2026-10-22) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py index 7ecd702a7a8b..de5c2b248123 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py @@ -4,8 +4,9 @@ import math import os import logging +import re import reprlib -from typing import Any, Dict, Union, List, Optional +from typing import Any, Dict, Union, List, Optional, Tuple from typing_extensions import overload, override from azure.ai.evaluation._exceptions import ( EvaluationException, @@ -26,7 +27,21 @@ _TEXT_CONTENT_TYPES = frozenset({"text", "input_text", "output_text"}) _CONVERSATION_ROLES = frozenset({"system", "developer", "user", "assistant"}) _TOOL_CALL_KEYS = ("tool_calls", "function_call") -_TOOL_TRACE_MARKERS = ("[TOOL_CALL]", "[TOOL_RESULT]") +# Tool calls and results written as text: this evaluator's [TOOL_CALL] and [TOOL_RESULT] markers, tags such as +# , [TOOL_CALLS] and , and the tool tokens of Llama and gpt-oss models. +_TOOL_TRACE_MARKER = re.compile( + r"tool_call|tool_result|tool_response|function_call|function_result|<\|python_tag\|>|", + re.IGNORECASE, +) +# A line that starts with a tool step label, as ReAct agents ("Action:", "Observation 2:") and LangChain's printed +# messages ("Tool Calls:") write it, also in a Markdown list, quote, heading or emphasis. The character classes hold +# no line breaks, so a search takes time linear in the length of the text. +_TOOL_STEP_LABEL = re.compile( + r"^[ \t>*_#+`.\d-]*" + r"(?:action|observation|(?:tool|function)[ _-]?(?:call|result|response|output)s?)" + r"[ \t\d*_#.()`]*:", + re.IGNORECASE | re.MULTILINE, +) _APPROVAL_TYPES = frozenset({"mcp_approval_request", "mcp_approval_response"}) @@ -151,7 +166,12 @@ def __call__( A list of messages that holds only text has no tool calls, so it is reported as not applicable without calling the judge model. A tool_calls, function_call or tool_call_id field that is None or empty does not - count as a tool call. + count as a tool call. A list of text messages that looks like a tool transcript still goes to the judge model, + as text with each message's role and every tool definition: text that contains tool_call, tool_result, + tool_response, function_call or function_result in any letter case, as tags such as and + [TOOL_CALLS] do, Llama's <|python_tag|> or token, or a line that starts + with a label such as Action:, Observation:, Tool call: or Tool result:, as ReAct agents write. Pass a transcript + in any other text format as a string so the judge model assesses it. :keyword response: The response being evaluated, either a string or a list of messages (full agent response potentially including tool calls) @@ -213,12 +233,16 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[str, float]]: # t target=ErrorTarget.TOOL_CALL_SUCCESS_EVALUATOR, ) + transcript = None if isinstance(eval_input.get("response"), list): # Decided before preprocessing, which drops whole messages that hold an MCP approval. if _has_no_tool_calls(eval_input["response"]): return self._return_not_applicable_result(_NO_TOOL_CALLS_MESSAGE, self._threshold) eval_input["response"] = _preprocess_messages(eval_input["response"]) - eval_input["tool_calls"] = _reformat_tool_calls_results(eval_input["response"], logger) + transcript = _tool_transcript(eval_input["response"]) + eval_input["tool_calls"] = ( + transcript if transcript is not None else _reformat_tool_calls_results(eval_input["response"], logger) + ) # If response is a string, pass directly without reformatting elif isinstance(eval_input["response"], str): eval_input["tool_calls"] = eval_input["response"] @@ -236,8 +260,8 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[str, float]]: # t # If tool definitions are string, pass directly without reformatting, else format it. if "tool_definitions" in eval_input and not isinstance(eval_input["tool_definitions"], str): tool_definitions = eval_input["tool_definitions"] - # Only if response is not a string, we filter tool definitions to only tools needed. - if not isinstance(eval_input["response"], str): + # Keep only the tools that were called, unless the calls are in text: a string or a transcript. + if not isinstance(eval_input["response"], str) and transcript is None: tool_definitions = _filter_to_used_tools( tool_definitions=tool_definitions, msgs_list=eval_input["response"], @@ -322,18 +346,19 @@ def _is_one_of(value: Any, names: frozenset) -> bool: return isinstance(value, str) and value in names -def _has_no_tool_calls(messages) -> bool: - """Return True only for a list of text messages and MCP approvals that holds no tool calls, results or traces. +def _texts_of_text_messages(messages) -> Optional[List[Tuple[str, str]]]: + """Return the role and text of each text in a list of text messages and MCP approvals, or None for any other value. A tool_calls, function_call or tool_call_id field that holds nothing, such as the None that OpenAI's model_dump() writes, counts as absent. A list in any other format may hold tool activity this evaluator does not parse, so the judge assesses it. """ if not isinstance(messages, list) or not messages: - return False + return None + texts = [] for message in messages: if not isinstance(message, dict) or not all(_holds_nothing(message.get(key)) for key in _TOOL_CALL_KEYS): - return False + return None content = message.get("content") items = content if isinstance(content, list): @@ -346,22 +371,46 @@ def _has_no_tool_calls(messages) -> bool: # Only MCP approval requests or responses, which are not tool calls; a response names its request # in tool_call_id. continue - if not _is_one_of(message.get("role"), _CONVERSATION_ROLES) or not _holds_nothing(message.get("tool_call_id")): - return False + role = message.get("role") + if not _is_one_of(role, _CONVERSATION_ROLES) or not _holds_nothing(message.get("tool_call_id")): + return None if isinstance(content, str): - texts = [content] + message_texts = [content] elif isinstance(items, list) and all( isinstance(item, dict) and _is_one_of(item.get("type"), _TEXT_CONTENT_TYPES) for item in items ): - texts = [item.get("text") for item in items] + message_texts = [item.get("text") for item in items] else: - return False - # Text can carry tool calls in this evaluator's own trace format. - if not all( - isinstance(text, str) and not any(marker in text for marker in _TOOL_TRACE_MARKERS) for text in texts - ): - return False - return True + return None + if not all(isinstance(text, str) for text in message_texts): + return None + texts.extend((role, text) for text in message_texts) + return texts + + +def _has_no_tool_calls(messages) -> bool: + """Return True only for a list of text messages and MCP approvals that holds no tool calls, results or traces. + + Text that looks like a tool transcript, such as ReAct steps or tags, counts as tool activity. + """ + texts = _texts_of_text_messages(messages) + return texts is not None and not any(_holds_tool_transcript(text) for _, text in texts) + + +def _tool_transcript(messages) -> Optional[str]: + """Return the text of a list of text messages that holds a tool transcript, with each message's role, or None. + + A Decision model can miss ReAct steps in a list of messages that it recognizes in the same text on its own. + """ + texts = _texts_of_text_messages(messages) + if texts is None or not any(_holds_tool_transcript(text) for _, text in texts): + return None + return "\n\n".join(f"{role}: {text}" for role, text in texts if text) + + +def _holds_tool_transcript(text: str) -> bool: + """Return True when the text records tool calls or results in a text format this evaluator recognizes.""" + return _TOOL_TRACE_MARKER.search(text) is not None or _TOOL_STEP_LABEL.search(text) is not None def _filter_to_used_tools(tool_definitions, msgs_list, logger=None): diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py index 82ce227c8ad9..60b8c7afd373 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py @@ -18,6 +18,7 @@ from azure.ai.evaluation._evaluators._tool_call_success._tool_call_success import ( _ToolCallSuccessEvaluator, _has_no_tool_calls, + _tool_transcript, ) from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException @@ -274,6 +275,7 @@ class TestToolCallSuccessWithoutToolCalls: [{"role": "assistant", "content": "Hello.", "function_call": None}], [{"role": "user", "content": "Hi.", "tool_call_id": None}, {"role": "assistant", "content": "Hello."}], [OPENAI_TEXT_REPLY], + [{"role": "assistant", "content": "Next action: wait for the forecast.\nAction items: none."}], ], ids=[ "text-items", @@ -285,6 +287,7 @@ class TestToolCallSuccessWithoutToolCalls: "null-function-call", "null-tool-call-id", "openai-model-dump", + "mentions-actions", ], ) def test_a_response_without_tool_calls_is_not_applicable_without_a_judge_call(self, config, response): @@ -386,9 +389,161 @@ def test_a_decision_judge_still_gets_openai_tool_calls(self): assert evaluator._decision.evaluate.await_count == 1 assert "fetch_weather" in str(evaluator._decision.evaluate.await_args.args[0]["tool_calls"]) + @pytest.mark.parametrize("shape", ["string", "text-item"]) + @pytest.mark.parametrize( + "transcript", + [ + "Thought: I need the weather.\nAction: fetch_weather\nAction Input: Paris\nObservation: It timed out.", + "Observation: It timed out.", + "Action: fetch_weather\nAction Input: Paris\nIt timed out.", + "Thought 1: I need the weather.\nAction 1: Search[Paris weather]\nObservation 1: It timed out.", + "**Action:** fetch_weather\n**Observation:** It timed out.", + "> observation: It timed out.", + "1. Action: fetch_weather\n2. Observation: It timed out.", + "Thought: Retrying.\r\nObservation: It timed out.\r\n", + "Tool Calls:\n fetch_weather (call_1)\n Call ID: call_1\n Args:\n city: Paris\nIt timed out.", + "Tool result: It timed out.", + "Function call: fetch_weather(city='Paris')\nIt timed out.", + '\n{"name": "fetch_weather", "arguments": {"city": "Paris"}}\n\nIt timed out.', + "\nIt timed out.\n", + '[TOOL_CALLS] [{"name": "fetch_weather", "arguments": {"city": "Paris"}}]\nIt timed out.', + "[TOOL_RESULTS] It timed out. [/TOOL_RESULTS]", + "[tool_call] fetch_weather(city=Paris)\n[tool_result] It timed out.", + '\n\n\nIt timed out.', + "\nIt timed out.\n", + '<|python_tag|>fetch_weather.call(city="Paris")\nIt timed out.', + '{"city": "Paris"}\nIt timed out.', + '<|channel|>commentary to=functions.fetch_weather<|message|>{"city": "Paris"}<|call|>\nIt timed out.', + "<|tool_calls_section_begin|><|tool_call_begin|>functions.fetch_weather:0\nIt timed out.", + ], + ids=[ + "react", + "react-observation-only", + "react-action-only", + "react-numbered-steps", + "react-markdown-bold", + "react-quoted-lowercase", + "react-numbered-list", + "react-crlf", + "langchain-printed-messages", + "tool-result-label", + "function-call-label", + "hermes-tool-call", + "hermes-tool-response", + "mistral-tool-calls", + "mistral-tool-results", + "lowercase-trace-markers", + "anthropic-function-calls", + "anthropic-function-results", + "llama-python-tag", + "llama-function-tag", + "gpt-oss-call", + "kimi-tool-call-tokens", + ], + ) + def test_a_tool_transcript_in_text_still_goes_to_the_judge(self, transcript, shape): + content = transcript if shape == "string" else [{"type": "output_text", "text": transcript}] + evaluator = judge({"score": 0, "status": "completed", "reason": "The tool call timed out."}) + + result = evaluator( + response=[{"role": "user", "content": "Weather in Paris?"}, {"role": "assistant", "content": content}] + ) + + assert evaluator._flow.await_count == 1 + assert "It timed out." in str(evaluator._flow.await_args.kwargs["tool_calls"]) + assert result["tool_call_success_result"] == "fail" + + def test_a_decision_judge_gets_a_tool_transcript_in_text(self): + evaluator = _ToolCallSuccessEvaluator(model_config=dict(DECISION_CONFIG)) + evaluator._decision.evaluate = AsyncMock(return_value={"tool_call_success": 0.0}) + transcript = "Action: fetch_weather\nAction Input: Paris\nObservation: It timed out." + + assert evaluator(response=[{"role": "assistant", "content": transcript}])["tool_call_success"] == 0.0 + assert evaluator._decision.evaluate.await_args.args[0]["tool_calls"] == f"assistant: {transcript}" + + def test_a_list_with_a_tool_transcript_goes_to_the_judge_as_text_with_each_role(self): + evaluator = judge({"score": 0, "status": "completed", "reason": "r"}) + response = [ + {"role": "system", "content": "Answer briefly."}, + {"role": "user", "content": [{"type": "input_text", "text": "Weather in Paris?"}]}, + {"role": "assistant", "content": [{"type": "output_text", "text": "Action: fetch_weather"}]}, + {"role": "user", "content": "Observation: It timed out.", "tool_calls": None}, + {"role": "assistant", "content": [{"type": "text", "text": ""}, {"type": "text", "text": "Sorry."}]}, + ] + + evaluator(response=response) + + assert evaluator._flow.await_args.kwargs["tool_calls"] == ( + "system: Answer briefly.\n\nuser: Weather in Paris?\n\nassistant: Action: fetch_weather\n\n" + "user: Observation: It timed out.\n\nassistant: Sorry." + ) + + def test_a_message_with_an_mcp_approval_is_left_out_of_a_transcript_as_it_is_from_every_judge_input(self): + evaluator = judge({"score": 0, "status": "completed", "reason": "r"}) + approval = {"type": "mcp_approval_request", "name": "search", "arguments": "{}"} + response = [ + {"role": "assistant", "content": [approval, {"type": "output_text", "text": "Action: search"}]}, + {"role": "assistant", "content": "Action: fetch_weather\nObservation: It timed out."}, + ] + + evaluator(response=response) + + assert evaluator._flow.await_args.kwargs["tool_calls"] == ( + "assistant: Action: fetch_weather\nObservation: It timed out." + ) + + @pytest.mark.parametrize("shape", ["string", "text-item"]) + def test_a_tool_transcript_keeps_every_tool_definition(self, shape): + transcript = "Action: fetch_weather\nObservation: {}" + content = transcript if shape == "string" else [{"type": "output_text", "text": transcript}] + definitions = [ + { + "name": "fetch_weather", + "description": "Returns the weather.", + "parameters": {"properties": {"city": {}}}, + }, + {"name": "send_email", "description": "Sends an email.", "parameters": {}}, + ] + evaluator = judge({"score": 0, "status": "completed", "reason": "r"}) + + evaluator(response=[{"role": "assistant", "content": content}], tool_definitions=definitions) + + assert evaluator._flow.await_args.kwargs["tool_definitions"] == ( + "TOOL_DEFINITIONS:\n- fetch_weather: Returns the weather. (inputs: city)\n" + "- send_email: Sends an email. (inputs: no parameters)" + ) + + @pytest.mark.parametrize( + "response", + [ + TOOL_CALL_RESPONSE, + TEXT_RESPONSE, + OPENAI_TOOL_CALL_RESPONSE, + [{"role": "assistant", "content": "Observation: It timed out.", "tool_calls": [{"id": "call_1"}]}], + [{"role": "tool", "content": "Observation: It timed out."}], + [ + { + "role": "assistant", + "content": [{"type": "text", "text": None}, {"type": "text", "text": ""}], + } + ], + "Observation: It timed out.", + [], + ], + ids=["tool-call", "text", "openai-tool-call", "tool-calls-field", "tool-role", "null-text", "string", "empty"], + ) + def test_only_text_messages_with_a_tool_transcript_become_a_transcript(self, response): + assert _tool_transcript(response) is None + def test_evaluate_leaves_rows_without_tool_calls_out_of_the_pass_rate(self, tmp_path): failed_call = json.loads(json.dumps(TOOL_CALL_RESPONSE).replace("sunny", "timed out")) - rows = [{"response": TOOL_CALL_RESPONSE}, {"response": failed_call}, {"response": [OPENAI_TEXT_REPLY]}] + rows = [ + {"response": TOOL_CALL_RESPONSE}, + {"response": failed_call}, + {"response": [OPENAI_TEXT_REPLY]}, + {"response": [{"role": "assistant", "content": "Action: fetch_weather\nObservation: It is sunny."}]}, + {"response": [{"role": "assistant", "content": "Action: fetch_weather\nObservation: It timed out."}]}, + ] data = tmp_path / "rows.jsonl" data.write_text("\n".join(json.dumps(row) for row in rows), encoding="utf-8") evaluator = _ToolCallSuccessEvaluator(model_config=dict(CHAT_CONFIG)) @@ -406,7 +561,7 @@ async def flow(timeout, **kwargs): ) results = sorted(row["outputs.tool_call_success.tool_call_success_result"] for row in result["rows"]) - assert results == ["fail", "not_applicable", "pass"] + assert results == ["fail", "fail", "not_applicable", "pass", "pass"] assert result["metrics"]["tool_call_success.tool_call_success"] == 0.5 assert result["metrics"]["tool_call_success.binary_aggregate"] == 0.5 @@ -474,6 +629,29 @@ def test_text_messages_have_no_tool_calls(self): assert _has_no_tool_calls(TEXT_RESPONSE) is True assert _has_no_tool_calls([{"role": "assistant", "content": []}]) is True + @pytest.mark.parametrize( + "text", + [ + "The next action: wait for the forecast.", + "Reaction: the user was happy.", + "Transaction: 42", + "Actionable: yes", + "Action items: none.", + "Observations: none.", + "Toolbox: a hammer.", + "Tools: none.", + "Calling a tool would not help here.", + ], + ) + def test_text_that_only_mentions_actions_or_tools_has_no_tool_calls(self, text): + assert _has_no_tool_calls([{"role": "assistant", "content": text}]) is True + assert _has_no_tool_calls([{"role": "assistant", "content": [{"type": "text", "text": text}]}]) is True + + @pytest.mark.parametrize("role", ["system", "developer", "user", "assistant"]) + def test_a_tool_transcript_in_any_text_message_counts(self, role): + assert _has_no_tool_calls([{"role": role, "content": "Observation: It timed out."}]) is False + assert _has_no_tool_calls([{"role": role, "content": [{"type": "input_text", "text": ""}]}]) is False + @pytest.mark.parametrize("key", ["tool_calls", "function_call", "tool_call_id"]) @pytest.mark.parametrize("value", [None, "", [], {}], ids=["none", "empty-string", "empty-list", "empty-dict"]) def test_a_call_field_that_holds_nothing_counts_as_absent(self, key, value): From da337c5d7ead81f0a8cdb3bffa5afda510a9a378 Mon Sep 17 00:00:00 2001 From: Ahmad Qardahji Date: Wed, 4 Nov 2026 17:17:56 -0800 Subject: [PATCH 2/5] Explain Decision skips in plain words A Decision evaluator that the judge model rates as not applicable gave the reason "Status completion probability 0.2000 is below 0.5000; evaluation skipped.", which names an internal question. After the "Not applicable: " prefix it now reads "the judge model's applicability probability 0.2000 is below the 0.5000 cutoff, so the evaluator was skipped.", with the same numbers. --- sdk/evaluation/azure-ai-evaluation/CHANGELOG.md | 1 + .../ai/evaluation/_evaluators/_common/_decision_strategy.py | 4 ++-- .../tests/unittests/test_builtin_decision_evaluators.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index b3fe3c4a940e..59d43aa97dfc 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -16,6 +16,7 @@ - `_ToolCallSuccessEvaluator` reports a list of messages that holds no tool calls, such as one that contains only text or only MCP approval messages, as not applicable without calling the chat or Decision judge model. A `tool_calls`, `function_call` or `tool_call_id` field that is None or empty, as in OpenAI's `model_dump()` of a reply without tool calls, does not count as a tool call. Strings, messages with any other content, and text that looks like a tool transcript are still sent to the judge model. Such text contains `tool_call`, `tool_result`, `tool_response`, `function_call` or `function_result` in any letter case, as tags such as ``, `[TOOL_CALLS]` and this evaluator's `[TOOL_RESULT]` do, Llama's `<|python_tag|>` or `` token, or has a line that starts with a label such as `Action:`, `Observation:`, `Tool call:` or `Tool result:`, as ReAct agents write. A list of text messages with such text is sent as its text, with each message's role and every tool definition, as a string is, because a Decision model can miss ReAct steps inside a list of messages. Pass a transcript in any other text format as a string for the judge model to assess it. - `evaluate()` computes `.binary_aggregate` over the rows whose result is `pass` or `fail`. Other rows, such as not applicable or errored rows, used to count as failures, although the mean score leaves them out. An evaluator without `pass` or `fail` rows, including a custom evaluator whose results have other values, has no `binary_aggregate`, as one without scores has no mean. With several result columns, the last one whose values can be counted decides, as before, so an evaluator whose `pass` and `fail` rows are only in an earlier column also has no `binary_aggregate`. - An evaluator's inputs keep the order in which its `__call__` overloads declare them, instead of the order of a Python `set`, which varies with the process's hash seed. An evaluator that sends these inputs to a Decision model unchanged, as several Microsoft Foundry catalog evaluators do, could list the same fields in a different order in each process, and the Decision model could answer the same input differently. The built-in evaluators in this package already send their Decision inputs in a fixed order. +- A Decision evaluator that the judge model rates as not applicable gives a readable reason, for example `Not applicable: the judge model's applicability probability 0.2000 is below the 0.5000 cutoff, so the evaluator was skipped. No model rationale is available.` It used to read `Status completion probability 0.2000 is below 0.5000; evaluation skipped.`, which named an internal question. ## 1.18.10 (2026-10-22) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py index 1f75f1d464a8..cdc34804015a 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py @@ -57,8 +57,8 @@ def decision_skip_reason( if probability >= cutoff: return None return ( - f"Status completion probability {probability:.4f} is below {cutoff:.4f}; " - "evaluation skipped. No model rationale is available." + f"the judge model's applicability probability {probability:.4f} is below the {cutoff:.4f} cutoff, " + "so the evaluator was skipped. No model rationale is available." ) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_builtin_decision_evaluators.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_builtin_decision_evaluators.py index 5c1e82fec93e..993e1704fa29 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_builtin_decision_evaluators.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_builtin_decision_evaluators.py @@ -554,8 +554,8 @@ def test_status_false_is_not_applicable(service, evaluator_type, key, rubric, in f"{key}_score": None, f"{key}_passed": None, f"{key}_result": "not_applicable", - f"{key}_reason": "Not applicable: Status completion probability 0.2000 is below 0.5000; evaluation skipped. " - "No model rationale is available.", + f"{key}_reason": "Not applicable: the judge model's applicability probability 0.2000 is below the 0.5000 " + "cutoff, so the evaluator was skipped. No model rationale is available.", f"{key}_status": "skipped", f"{key}_threshold": threshold, f"{key}_prompt_tokens": 40, From f593bb982db384dc1bdaacce69302cf709f4c15b Mon Sep 17 00:00:00 2001 From: Ahmad Qardahji Date: Wed, 4 Nov 2026 20:24:28 -0800 Subject: [PATCH 3/5] Move this change's notes to 1.18.12 and bump the version 1.18.11 shipped without this change, so its transcript handling and readable skip reason are listed under 1.18.12 (Unreleased), and the 1.18.11 Tool Call Success entry again describes what 1.18.11 does. The wording fixes to three other 1.18.11 entries stay there, since they describe 1.18.11. --- sdk/evaluation/azure-ai-evaluation/CHANGELOG.md | 16 ++++++++++++++-- .../azure/ai/evaluation/_version.py | 2 +- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index 14921f9b686a..4c477b709e75 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -1,5 +1,18 @@ # Release History +## 1.18.12 (Unreleased) + +### Features Added + +### Breaking Changes + +### Bugs Fixed + +- `_ToolCallSuccessEvaluator` sends a list of text messages to the judge model, rather than reporting it as not applicable, when its text looks like a tool transcript. Such text contains `tool_call`, `tool_result`, `tool_response`, `function_call` or `function_result` in any letter case, as tags such as ``, `[TOOL_CALLS]` and this evaluator's `[TOOL_RESULT]` do, Llama's `<|python_tag|>` or `` token, or has a line that starts with a label such as `Action:`, `Observation:`, `Tool call:` or `Tool result:`, as ReAct agents write. In 1.18.11 only text with `[TOOL_CALL]` or `[TOOL_RESULT]` was sent. Such a list is sent as its text, with each message's role and every tool definition, as a string is, because a Decision model can miss ReAct steps inside a list of messages. Pass a transcript in any other text format as a string for the judge model to assess it. +- A Decision evaluator that the judge model rates as not applicable gives a readable reason, for example `Not applicable: the judge model's applicability probability 0.2000 is below the 0.5000 cutoff, so the evaluator was skipped. No model rationale is available.` It used to read `Status completion probability 0.2000 is below 0.5000; evaluation skipped.`, which named an internal question. + +### Other Changes + ## 1.18.11 (2026-11-03) ### Features Added @@ -13,10 +26,9 @@ ### Bugs Fixed - `_ToolCallSuccessEvaluator` no longer scores successful tool calls 0 when a chat judge model, such as gpt-4.1 or gpt-4o, puts `score` and `status` inside `properties`. The example outputs in its prompt are now valid JSON, and `score`, `status` and `reason` are read from `properties` when the output has none at the top level. A missing score is no longer reported as 0, and a score that does not read as 0 or 1 raises a system-classified `EvaluationException`. Values that read as 0 or 1, such as `True`, `"1"` and `1.0`, are still accepted. -- `_ToolCallSuccessEvaluator` reports a list of messages that holds no tool calls, such as one that contains only text or only MCP approval messages, as not applicable without calling the chat or Decision judge model. A `tool_calls`, `function_call` or `tool_call_id` field that is None or empty, as in OpenAI's `model_dump()` of a reply without tool calls, does not count as a tool call. Strings, messages with any other content, and text that looks like a tool transcript are still sent to the judge model. Such text contains `tool_call`, `tool_result`, `tool_response`, `function_call` or `function_result` in any letter case, as tags such as ``, `[TOOL_CALLS]` and this evaluator's `[TOOL_RESULT]` do, Llama's `<|python_tag|>` or `` token, or has a line that starts with a label such as `Action:`, `Observation:`, `Tool call:` or `Tool result:`, as ReAct agents write. A list of text messages with such text is sent as its text, with each message's role and every tool definition, as a string is, because a Decision model can miss ReAct steps inside a list of messages. Pass a transcript in any other text format as a string for the judge model to assess it. +- `_ToolCallSuccessEvaluator` reports a list of messages that holds no tool calls, such as one that contains only text or only MCP approval messages, as not applicable without calling the chat or Decision judge model. A `tool_calls`, `function_call` or `tool_call_id` field that is None or empty, as in OpenAI's `model_dump()` of a reply without tool calls, does not count as a tool call. Strings, text that contains `[TOOL_CALL]` or `[TOOL_RESULT]`, and messages with any other content are still sent to the judge model. - `evaluate()` computes `.binary_aggregate` over the rows whose result is `pass` or `fail`. Other rows, such as not applicable or errored rows, used to count as failures, although the mean score leaves them out. An evaluator without `pass` or `fail` rows, including a custom evaluator whose results have other values, has no `binary_aggregate`, as one without scores has no mean. With several result columns, the last one whose values can be counted decides, as before, so an evaluator whose `pass` and `fail` rows are only in an earlier column also has no `binary_aggregate`. - An evaluator's inputs keep the order in which its `__call__` overloads declare them, instead of the order of a Python `set`, which varies with the process's hash seed. An evaluator that sends these inputs to a Decision model unchanged, as several Microsoft Foundry catalog evaluators do, could list the same fields in a different order in each process, and the Decision model could answer the same input differently. The built-in evaluators in this package already send their Decision inputs in a fixed order. -- A Decision evaluator that the judge model rates as not applicable gives a readable reason, for example `Not applicable: the judge model's applicability probability 0.2000 is below the 0.5000 cutoff, so the evaluator was skipped. No model rationale is available.` It used to read `Status completion probability 0.2000 is below 0.5000; evaluation skipped.`, which named an internal question. ## 1.18.10 (2026-10-22) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_version.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_version.py index 03d429c451a0..a307b7b58822 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_version.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_version.py @@ -3,4 +3,4 @@ # --------------------------------------------------------- # represents upcoming version -VERSION = "1.18.11" +VERSION = "1.18.12" From e9fb6124c217148621f5deea09f71b44c85df31e Mon Sep 17 00:00:00 2001 From: Ahmad Qardahji Date: Thu, 5 Nov 2026 18:50:03 -0800 Subject: [PATCH 4/5] Report Task adherence's pass threshold and drop key namespaces from Decision reasons TaskAdherenceEvaluator with a Decision model reported threshold 0 by default although its binary result passes only at 1; it now reports 1, as _TaskCompletionEvaluator does. A requested threshold is still reported as given. Decision reasons named a question by its whole key, so the hosted service's rubric keys such as grade/confirm_details read "Grade/confirm details probability ...". They now use the part after the last slash. --- .../azure-ai-evaluation/CHANGELOG.md | 2 + .../_evaluators/_common/_decision_strategy.py | 4 +- .../_task_adherence/_task_adherence.py | 3 +- .../test_builtin_decision_evaluators.py | 55 +++++++++++++++++-- 4 files changed, 56 insertions(+), 8 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index 4c477b709e75..e06898e5d6f0 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -10,6 +10,8 @@ - `_ToolCallSuccessEvaluator` sends a list of text messages to the judge model, rather than reporting it as not applicable, when its text looks like a tool transcript. Such text contains `tool_call`, `tool_result`, `tool_response`, `function_call` or `function_result` in any letter case, as tags such as ``, `[TOOL_CALLS]` and this evaluator's `[TOOL_RESULT]` do, Llama's `<|python_tag|>` or `` token, or has a line that starts with a label such as `Action:`, `Observation:`, `Tool call:` or `Tool result:`, as ReAct agents write. In 1.18.11 only text with `[TOOL_CALL]` or `[TOOL_RESULT]` was sent. Such a list is sent as its text, with each message's role and every tool definition, as a string is, because a Decision model can miss ReAct steps inside a list of messages. Pass a transcript in any other text format as a string for the judge model to assess it. - A Decision evaluator that the judge model rates as not applicable gives a readable reason, for example `Not applicable: the judge model's applicability probability 0.2000 is below the 0.5000 cutoff, so the evaluator was skipped. No model rationale is available.` It used to read `Status completion probability 0.2000 is below 0.5000; evaluation skipped.`, which named an internal question. +- `TaskAdherenceEvaluator` with a Decision model reports a threshold of 1 when none is requested, the only score at which its binary result passes, as `_TaskCompletionEvaluator` does. It reported 0. A requested threshold is still reported as given and still does not decide whether a binary result passes. The evaluator's `threshold` attribute now also defaults to 1 for chat models; chat results are unchanged. +- A Decision reason names a question by the part of its key after the last `/`, so a namespaced key such as `grade/confirm_details`, which the hosted evaluation service uses for rubric dimensions, reads `Confirm details probability 0.1645 is below the 0.7500 cutoff.` It used to read `Grade/confirm details probability ...`. ### Other Changes diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py index cdc34804015a..4533f9e837fe 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py @@ -74,7 +74,9 @@ def decision_token_counts(response: Dict[str, Any]) -> Dict[str, Any]: def _label(result_key: str) -> str: - return result_key.replace("_", " ") + # A caller can namespace a question key, as in "grade/confirm_details"; the label names only the question. + leaf = result_key.rsplit("/", 1)[-1] + return (leaf or result_key).replace("_", " ") def _digest(question: Dict[str, Any]) -> str: diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_task_adherence/_task_adherence.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_task_adherence/_task_adherence.py index 85bfe9c6642f..fe324827f5c3 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_task_adherence/_task_adherence.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_task_adherence/_task_adherence.py @@ -76,7 +76,8 @@ class TaskAdherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]): _supports_decision_models = True _binary_decision = True - _DEFAULT_TASK_ADHERENCE_SCORE = 0 + # A binary result passes only at a score of 1, so Decision results report 1 unless a threshold is requested. + _DEFAULT_TASK_ADHERENCE_SCORE = 1 _validator: ValidatorInterface diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_builtin_decision_evaluators.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_builtin_decision_evaluators.py index 993e1704fa29..cbdb8e6d820c 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_builtin_decision_evaluators.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_builtin_decision_evaluators.py @@ -49,7 +49,7 @@ from azure.ai.evaluation._evaluators._task_completion import _TaskCompletionEvaluator from azure.ai.evaluation._evaluators._tool_input_accuracy import _ToolInputAccuracyEvaluator from azure.ai.evaluation._evaluators._tool_selection import _ToolSelectionEvaluator -from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, EvaluationException +from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException pytestmark = pytest.mark.unittest @@ -186,7 +186,7 @@ "task_adherence.decision.yaml", {"query": QUERY, "response": RESPONSE}, {"system_message": "", "query": QUERY, "response": RESPONSE, "tool_calls": ""}, - 0, + 1, ), ( _TaskCompletionEvaluator, @@ -581,11 +581,54 @@ def test_construction_selects_the_strategy_from_the_model_type(service, evaluato assert decision._decision.timeout == 120 and decision._decision.threshold == decision._threshold -@pytest.mark.parametrize("evaluator_type", [TaskAdherenceEvaluator, _TaskCompletionEvaluator]) -def test_a_requested_threshold_is_reported_on_decision_models(service, evaluator_type): - decision = _decision(evaluator_type, threshold=2) +@pytest.mark.parametrize("requested", [0, 2]) +@pytest.mark.parametrize( + "evaluator_type,key", [(TaskAdherenceEvaluator, "task_adherence"), (_TaskCompletionEvaluator, "task_completion")] +) +def test_a_requested_threshold_is_reported_on_decision_models(service, evaluator_type, key, requested): + decision = _decision(evaluator_type, threshold=requested) + + assert decision._threshold == decision._decision.threshold == requested + result = decision(query=QUERY, response=RESPONSE) + assert result[f"{key}_threshold"] == requested and result[f"{key}_passed"] is True + + +def test_task_adherence_chat_threshold_is_unchanged(): + with patch.object(_model_runners.AsyncPrompty, "load"): + chat = TaskAdherenceEvaluator(deepcopy(CHAT_CONFIG)) + + assert chat._threshold == 3.0 and chat.threshold == 1 + + +@pytest.mark.parametrize( + "result_key,label", + [ + ("grade/confirm_details", "confirm details"), + ("a/b/c_d", "c d"), + ("task_completion", "task completion"), + ("grade/", "grade/"), + ("/", "/"), + ("", ""), + ], +) +def test_a_namespaced_question_key_is_labelled_by_its_question(result_key, label): + noul = {"answers": {result_key: {"type": "noul", "noul": 0.1645}}} + scored = { + "answers": { + result_key: { + "type": "score", + "score": sum(level * value for level, value in enumerate(LEVELS)), + "confidence": max(LEVELS), + "probabilities": {str(level): value for level, value in enumerate(LEVELS)}, + } + } + } + + _, binary_reason, _ = BinaryScoring(0.75).interpret(noul, result_key, ErrorTarget.EVALUATE) + _, likert_reason, _ = LikertScoring(5, minimum=1).interpret(scored, result_key, ErrorTarget.EVALUATE) - assert decision._threshold == decision._decision.threshold == 2 + assert binary_reason == f"{label.capitalize()} probability 0.1645 is below the 0.7500 cutoff." + assert likert_reason == f"Selected {label} level 4 from the highest-probability level." def test_groundedness_chat_construction_has_no_decision_strategies(): From 93ec9a0a0d59d14719d15d18938f05c627f8ce0d Mon Sep 17 00:00:00 2001 From: Ahmad Qardahji Date: Fri, 6 Nov 2026 11:09:03 -0800 Subject: [PATCH 5/5] Recognize list-marked tool steps and skip MCP approval message text in Tool Call Success A line such as "1) Action:", "(2) Observation:", "a) Action:" or a bulleted or no-break-space indented label, and a label with a fullwidth colon, now count as a tool transcript. The text of a message that preprocessing drops for holding an MCP approval request no longer makes a list applicable, because no judge input includes it. Structured tool calls beside an approval still go to the judge. --- .../azure-ai-evaluation/CHANGELOG.md | 2 +- .../_tool_call_success/_tool_call_success.py | 20 ++++++----- .../test_tool_call_success_evaluator.py | 36 +++++++++++++++++++ 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index e06898e5d6f0..1e8ceb062f45 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -8,7 +8,7 @@ ### Bugs Fixed -- `_ToolCallSuccessEvaluator` sends a list of text messages to the judge model, rather than reporting it as not applicable, when its text looks like a tool transcript. Such text contains `tool_call`, `tool_result`, `tool_response`, `function_call` or `function_result` in any letter case, as tags such as ``, `[TOOL_CALLS]` and this evaluator's `[TOOL_RESULT]` do, Llama's `<|python_tag|>` or `` token, or has a line that starts with a label such as `Action:`, `Observation:`, `Tool call:` or `Tool result:`, as ReAct agents write. In 1.18.11 only text with `[TOOL_CALL]` or `[TOOL_RESULT]` was sent. Such a list is sent as its text, with each message's role and every tool definition, as a string is, because a Decision model can miss ReAct steps inside a list of messages. Pass a transcript in any other text format as a string for the judge model to assess it. +- `_ToolCallSuccessEvaluator` sends a list of text messages to the judge model, rather than reporting it as not applicable, when its text looks like a tool transcript. Such text contains `tool_call`, `tool_result`, `tool_response`, `function_call` or `function_result` in any letter case, as tags such as ``, `[TOOL_CALLS]` and this evaluator's `[TOOL_RESULT]` do, Llama's `<|python_tag|>` or `` token, or has a line that starts with a label such as `Action:`, `Observation:`, `Tool call:` or `Tool result:`, as ReAct agents write, also after a bullet or a list marker such as `1)`, `(2)` or `a)`. In 1.18.11 only text with `[TOOL_CALL]` or `[TOOL_RESULT]` was sent. Such a list is sent as its text, with each message's role and every tool definition, as a string is, because a Decision model can miss ReAct steps inside a list of messages. The text of a message that holds an MCP approval request is not counted, because it is left out of every judge input. Pass a transcript in any other text format as a string for the judge model to assess it. - A Decision evaluator that the judge model rates as not applicable gives a readable reason, for example `Not applicable: the judge model's applicability probability 0.2000 is below the 0.5000 cutoff, so the evaluator was skipped. No model rationale is available.` It used to read `Status completion probability 0.2000 is below 0.5000; evaluation skipped.`, which named an internal question. - `TaskAdherenceEvaluator` with a Decision model reports a threshold of 1 when none is requested, the only score at which its binary result passes, as `_TaskCompletionEvaluator` does. It reported 0. A requested threshold is still reported as given and still does not decide whether a binary result passes. The evaluator's `threshold` attribute now also defaults to 1 for chat models; chat results are unchanged. - A Decision reason names a question by the part of its key after the last `/`, so a namespaced key such as `grade/confirm_details`, which the hosted evaluation service uses for rubric dimensions, reads `Confirm details probability 0.1645 is below the 0.7500 cutoff.` It used to read `Grade/confirm details probability ...`. diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py index de5c2b248123..d1e7b47a825e 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py @@ -17,7 +17,7 @@ from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase from azure.ai.evaluation._evaluators._common._decision_rubric import rubric_strategy, with_probability_threshold from azure.ai.evaluation._evaluators._common._validators import ToolDefinitionsValidator, ValidatorInterface -from azure.ai.evaluation._common.utils import _log_safe_summary, _stringify_tool_result +from azure.ai.evaluation._common.utils import _drop_mcp_approval_messages, _log_safe_summary, _stringify_tool_result from azure.ai.evaluation._common._experimental import experimental logger = logging.getLogger(__name__) @@ -34,12 +34,13 @@ re.IGNORECASE, ) # A line that starts with a tool step label, as ReAct agents ("Action:", "Observation 2:") and LangChain's printed -# messages ("Tool Calls:") write it, also in a Markdown list, quote, heading or emphasis. The character classes hold -# no line breaks, so a search takes time linear in the length of the text. +# messages ("Tool Calls:") write it, also in a Markdown list, quote, heading or emphasis, after a bullet or a list +# marker such as "1)", "(2)" or "a)", and with a fullwidth colon. The character classes hold no line breaks and the +# list marker is bounded, so a search takes time linear in the length of the text. _TOOL_STEP_LABEL = re.compile( - r"^[ \t>*_#+`.\d-]*" + r"^[ \t\u00a0>*_#+`.\u2022\d-]*(?:\(?(?:\d{1,3}|[a-z])\)[ \t\u00a0]*)?" r"(?:action|observation|(?:tool|function)[ _-]?(?:call|result|response|output)s?)" - r"[ \t\d*_#.()`]*:", + r"[ \t\d*_#.()`]*[:\uff1a]", re.IGNORECASE | re.MULTILINE, ) _APPROVAL_TYPES = frozenset({"mcp_approval_request", "mcp_approval_response"}) @@ -235,7 +236,8 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[str, float]]: # t transcript = None if isinstance(eval_input.get("response"), list): - # Decided before preprocessing, which drops whole messages that hold an MCP approval. + # Decided on the raw list, so a list of only MCP approvals stays not applicable; the text of a message + # that preprocessing drops for holding an MCP approval is not counted. if _has_no_tool_calls(eval_input["response"]): return self._return_not_applicable_result(_NO_TOOL_CALLS_MESSAGE, self._threshold) eval_input["response"] = _preprocess_messages(eval_input["response"]) @@ -351,7 +353,8 @@ def _texts_of_text_messages(messages) -> Optional[List[Tuple[str, str]]]: A tool_calls, function_call or tool_call_id field that holds nothing, such as the None that OpenAI's model_dump() writes, counts as absent. A list in any other format may hold tool activity this evaluator does not parse, so the - judge assesses it. + judge assesses it. The text of a message that preprocessing drops for holding an MCP approval is left out, as no + judge input includes it. """ if not isinstance(messages, list) or not messages: return None @@ -384,7 +387,8 @@ def _texts_of_text_messages(messages) -> Optional[List[Tuple[str, str]]]: return None if not all(isinstance(text, str) for text in message_texts): return None - texts.extend((role, text) for text in message_texts) + if _drop_mcp_approval_messages([message]): + texts.extend((role, text) for text in message_texts) return texts diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py index 60b8c7afd373..0a9d2dbb9564 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py @@ -276,6 +276,25 @@ class TestToolCallSuccessWithoutToolCalls: [{"role": "user", "content": "Hi.", "tool_call_id": None}, {"role": "assistant", "content": "Hello."}], [OPENAI_TEXT_REPLY], [{"role": "assistant", "content": "Next action: wait for the forecast.\nAction items: none."}], + [ + { + "role": "assistant", + "content": [ + {"type": "mcp_approval_request", "name": "search", "arguments": "{}"}, + {"type": "output_text", "text": "Action: search\nObservation: It timed out."}, + ], + } + ], + [ + {"role": "user", "content": "Weather in Paris?"}, + { + "role": "assistant", + "content": [ + {"type": "mcp_approval_request", "name": "search", "arguments": "{}"}, + {"type": "output_text", "text": "Action: search\nObservation: It timed out."}, + ], + }, + ], ], ids=[ "text-items", @@ -288,6 +307,8 @@ class TestToolCallSuccessWithoutToolCalls: "null-tool-call-id", "openai-model-dump", "mentions-actions", + "transcript-only-beside-an-mcp-approval", + "user-turn-and-transcript-beside-an-mcp-approval", ], ) def test_a_response_without_tool_calls_is_not_applicable_without_a_judge_call(self, config, response): @@ -400,6 +421,12 @@ def test_a_decision_judge_still_gets_openai_tool_calls(self): "**Action:** fetch_weather\n**Observation:** It timed out.", "> observation: It timed out.", "1. Action: fetch_weather\n2. Observation: It timed out.", + "1) Action: fetch_weather\n2) Observation: It timed out.", + "(1) Action: fetch_weather\n(2) Observation: It timed out.", + "a) Action: fetch_weather\nb) Observation: It timed out.", + "\u2022 Action: fetch_weather\n\u2022 Observation: It timed out.", + "\u00a0Action: fetch_weather\n\u00a0Observation: It timed out.", + "Action\uff1a fetch_weather\nObservation\uff1a It timed out.", "Thought: Retrying.\r\nObservation: It timed out.\r\n", "Tool Calls:\n fetch_weather (call_1)\n Call ID: call_1\n Args:\n city: Paris\nIt timed out.", "Tool result: It timed out.", @@ -424,6 +451,12 @@ def test_a_decision_judge_still_gets_openai_tool_calls(self): "react-markdown-bold", "react-quoted-lowercase", "react-numbered-list", + "react-parenthesized-number-list", + "react-enclosed-number-list", + "react-lettered-list", + "react-bullet-list", + "react-no-break-space-indent", + "react-fullwidth-colon", "react-crlf", "langchain-printed-messages", "tool-result-label", @@ -641,6 +674,9 @@ def test_text_messages_have_no_tool_calls(self): "Toolbox: a hammer.", "Tools: none.", "Calling a tool would not help here.", + "(action: see below)", + "\u2014 action: none needed.", + "a) The next action: wait.", ], ) def test_text_that_only_mentions_actions_or_tools_has_no_tool_calls(self, text):