From 4c96b29518d234feb3690c839145de6f4011191d Mon Sep 17 00:00:00 2001 From: Ahmad Qardahji Date: Mon, 2 Nov 2026 19:22:49 -0800 Subject: [PATCH 01/14] Reuse credential tokens across model requests and fix Tool Call Success scoring Token reuse (from #49695, without the OpenAI Decisions protocol): - Decision requests, and Azure OpenAI chat requests that use Microsoft Entra ID, reuse a credential's token until five minutes before it expires instead of requesting one per request. - Concurrent requests that share a synchronous credential wait for one token request, off the event loop; requests queued behind a token that is not cached use it. - A 401 drops the token, and an acquisition that began before the rejection is not stored. - The default chat credential shares one credential and token across concurrent first requests. - A Decision credential that returns no string token is a user error, not a timeout. Tool Call Success: - The prompt's 14 example outputs are now valid JSON, which stops gpt-4.1 and gpt-4o nesting score and status in properties and scoring successful tool calls 0. - score, status and reason fall back to properties, and a score other than 0 or 1 raises a system error instead of becoming 0. - A list of messages without tool calls (only text or MCP approvals) is not applicable without a judge call; strings, tool traces in text and any other content still go to the judge. --- .../azure-ai-evaluation/CHANGELOG.md | 6 + .../_evaluators/_common/_model_runners.py | 72 +- .../_tool_call_success/_tool_call_success.py | 100 ++- .../tool_call_success.prompty | 28 +- .../_legacy/_common/_async_token_provider.py | 64 +- .../_legacy/_common/_token_cache.py | 231 +++++++ .../ai/evaluation/_legacy/prompty/_prompty.py | 99 ++- .../unittests/test_decision_model_support.py | 525 ++++++++++++++ .../unittests/test_prompty_token_reuse.py | 647 ++++++++++++++++++ .../test_tool_call_success_evaluator.py | 387 +++++++++++ 10 files changed, 2091 insertions(+), 68 deletions(-) create mode 100644 sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_token_cache.py create mode 100644 sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_token_reuse.py create mode 100644 sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index a69fabca1246..4d1624a44789 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -9,6 +9,12 @@ - The binary evaluators (task adherence, task completion, tool call success, tool input accuracy, tool output utilization and tool selection) accept a `probability_threshold` keyword. For Decision models it sets the probability at or above which the score is 1 (default 0.75) and overrides the value in `model_config`; chat models ignore it with a warning. - Conversation inputs are handled as before with a Decision model: evaluators that accept them score each user and assistant turn separately and aggregate the results. The catalog's whole-conversation rubrics are not included. +### Bugs Fixed + +- Decision requests, and Azure OpenAI chat model requests that authenticate with Microsoft Entra ID, reuse a credential's token until five minutes before it expires, as azure-core's bearer token policy does, instead of requesting a token for every request. Concurrent evaluations that share a synchronous credential wait for one token request, without blocking their event loops. With developer credentials such as `AzureCliCredential`, which start a process per request, concurrent evaluations were slow and could fail. Decision requests that waited on a failed token request share its failure instead of each requesting a token again, and chat requests share an authentication failure the same way. A token rejected with HTTP 401 is dropped and requested again; a token whose request began before the rejection is not kept for reuse, and requests that waited on it use it rather than each requesting another one. The default credential that a chat evaluator creates when no `credential` is passed is shared the same way. Tokens are kept per scope, and the cache never keeps a credential alive. A token without an expiry, or that expires within five minutes, is not kept, but the requests that waited for it use it. +- `_ToolCallSuccessEvaluator` no longer scores successful tool calls 0 when a chat judge model, such as gpt-4.1 or gpt-4o, puts `score` and `status` inside `properties`. The example outputs in its prompt are now valid JSON, and `score`, `status` and `reason` are read from `properties` when the output has none at the top level. A missing score is no longer reported as 0, and any score other than 0 or 1 raises a system-classified `EvaluationException`, as `_ToolInputAccuracyEvaluator` does. +- `_ToolCallSuccessEvaluator` reports a list of messages that holds no tool calls, such as one that contains only text or only MCP approval messages, as not applicable without calling the chat or Decision judge model. Strings, text that contains `[TOOL_CALL]` or `[TOOL_RESULT]`, and messages with any other content are still sent to the judge model. + ## 1.18.10 (2026-10-22) ### Features Added diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py index e2d34742c296..02acc4920972 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py @@ -16,6 +16,7 @@ import math import os import re +import threading import time from typing import Any, Callable, Dict, List, Optional from urllib.parse import urlsplit @@ -24,6 +25,7 @@ from azure.core.pipeline.transport import AioHttpTransport from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException +from azure.ai.evaluation._legacy._common._token_cache import TOKENS as _TOKENS from ..._common.utils import construct_prompty_model_config, validate_model_config from ..._http_utils import get_async_http_client @@ -340,6 +342,26 @@ def _is_finite_number(value: Any) -> bool: return False +def _token_failure(error: BaseException) -> Dict[str, Any]: + """Describe a failed token request without keeping the error, whose traceback references the credential.""" + if isinstance(error, EvaluationException): + return { + "message": error.message, + "internal_message": error.internal_message, + "target": error.target, + "category": error.category, + "blame": error.blame, + } + message = "Could not acquire a token for the decision model." + return { + "message": message, + "internal_message": message, + "target": ErrorTarget.EVALUATE, + "category": ErrorCategory.FAILED_EXECUTION, + "blame": ErrorBlame.USER_ERROR, + } + + def _probability(value: Any) -> float: if not _is_finite_number(value): raise ValueError("Decision probabilities must be finite numbers.") @@ -648,25 +670,56 @@ async def _await_request(self, content: bytes, deadline: float, execution: Decis task.cancel() await asyncio.gather(task, return_exceptions=True) - async def _get_token(self) -> str: - if inspect.iscoroutinefunction(self._credential.get_token): - return (await self._credential.get_token(_DECISION_TOKEN_SCOPE)).token - token = await asyncio.to_thread(self._credential.get_token, _DECISION_TOKEN_SCOPE) - if inspect.isawaitable(token): - token = await token - return token.token + async def _get_token(self, deadline: float) -> Optional[str]: + """Return a bearer token, or None when the deadline passed while waiting for another request's token.""" + credential = self._credential + token = _TOKENS.cached(credential, _DECISION_TOKEN_SCOPE) + if token is None: + # An acquisition that begins before a service rejects a token may return that token; the generation it + # began under lets the cache refuse it. + generation = _TOKENS.generation(credential, _DECISION_TOKEN_SCOPE) + if inspect.iscoroutinefunction(credential.get_token): + token = await credential.get_token(_DECISION_TOKEN_SCOPE) + _TOKENS.store(credential, _DECISION_TOKEN_SCOPE, token, generation) + else: + abandoned = threading.Event() + try: + token = await asyncio.to_thread( + _TOKENS.fetch, + credential, + _DECISION_TOKEN_SCOPE, + abandoned=abandoned, + deadline=deadline, + describe_failure=_token_failure, + ) + finally: + abandoned.set() + if token is None and time.monotonic() >= deadline: + return None + if inspect.isawaitable(token): + token = await token + _TOKENS.store(credential, _DECISION_TOKEN_SCOPE, token, generation) + value = getattr(token, "token", None) + if not isinstance(value, str): + raise _failed("Could not acquire a token for the decision model.", ErrorBlame.USER_ERROR) + return value async def _post(self, content: bytes, *, deadline: float, execution: DecisionExecution) -> Any: headers = {"x-ms-useragent": self._user_agent, "Content-Type": "application/json"} + token = None if self._api_key: headers["api-key"] = self._api_key else: try: - headers["Authorization"] = f"Bearer {await self._get_token()}" + token = await self._get_token(deadline) except EvaluationException: raise except Exception: # pylint: disable=broad-except raise _failed("Could not acquire a token for the decision model.", ErrorBlame.USER_ERROR) from None + if token is None: + execution.outcome = "deadline" + raise _failed("The decision model request timed out.") + headers["Authorization"] = "Bearer " + token policies: Dict[str, Any] = { "custom_hook_policy": execution, @@ -718,6 +771,9 @@ async def _post(self, content: bytes, *, deadline: float, execution: DecisionExe raise _failed("The decision model returned invalid JSON.") from None payload = None if status != 200: + if status == 401 and token is not None: + # A rejected token is not reused. + _TOKENS.discard(self._credential, _DECISION_TOKEN_SCOPE, token) blame = ErrorBlame.USER_ERROR if status in (400, 401, 403, 404, 422) else ErrorBlame.SYSTEM_ERROR provider_error = payload.get("error") if isinstance(payload, dict) else None code = provider_error.get("code") if isinstance(provider_error, dict) else None diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py index b098674aa189..127b5c9052ad 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py @@ -4,7 +4,8 @@ import math import os import logging -from typing import Dict, Union, List, Optional +import reprlib +from typing import Any, Dict, Union, List, Optional from typing_extensions import overload, override from azure.ai.evaluation._exceptions import ( EvaluationException, @@ -20,6 +21,14 @@ logger = logging.getLogger(__name__) +_NO_TOOL_CALLS_MESSAGE = "No tool calls found in response." +# Message content without tool activity. Any other content type may be a tool call or result. +_TEXT_CONTENT_TYPES = frozenset({"text", "input_text", "output_text"}) +_CONVERSATION_ROLES = frozenset({"system", "developer", "user", "assistant"}) +_TOOL_CALL_KEYS = ("tool_calls", "function_call") +_TOOL_TRACE_MARKERS = ("[TOOL_CALL]", "[TOOL_RESULT]") +_APPROVAL_TYPES = frozenset({"mcp_approval_request", "mcp_approval_response"}) + @experimental class _ToolCallSuccessEvaluator(PromptyEvaluatorBase[Union[str, float]]): @@ -131,11 +140,18 @@ def __call__( Example with list of messages: evaluator = _ToolCallSuccessEvaluator(model_config) - response = [{'createdAt': 1700000070, 'run_id': '0', 'role': 'assistant', - 'content': [{'type': 'text', 'text': '**Day 1:** Morning: Visit Louvre Museum (9 AM - 12 PM)...'}]}] + response = [ + {'role': 'assistant', 'content': [{'type': 'tool_call', 'tool_call_id': 'call_1', + 'name': 'fetch_weather', 'arguments': {'location': 'Paris'}}]}, + {'role': 'tool', 'tool_call_id': 'call_1', + 'content': [{'type': 'tool_result', 'tool_result': {'temperature': '20C'}}]}, + {'role': 'assistant', 'content': [{'type': 'text', 'text': 'It is 20C in Paris.'}]}] result = evaluator(response=response, ) + A list of messages that holds only text has no tool calls, so it is reported as not applicable without + calling the judge model. + :keyword response: The response being evaluated, either a string or a list of messages (full agent response potentially including tool calls) :paramtype response: Union[str, List[dict]] @@ -197,6 +213,9 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[str, float]]: # t ) if isinstance(eval_input.get("response"), list): + # Decided before preprocessing, which drops whole messages that hold an MCP approval. + if _has_no_tool_calls(eval_input["response"]): + return self._return_not_applicable_result(_NO_TOOL_CALLS_MESSAGE, self._threshold) eval_input["response"] = _preprocess_messages(eval_input["response"]) eval_input["tool_calls"] = _reformat_tool_calls_results(eval_input["response"], logger) # If response is a string, pass directly without reformatting @@ -235,17 +254,25 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[str, float]]: # t llm_output = prompty_output_dict.get("llm_output", prompty_output_dict) if isinstance(llm_output, dict): + properties = llm_output.get("properties") + llm_properties = properties if isinstance(properties, dict) else {} + reason = _top_level_or_nested(llm_output, llm_properties, "reason", "") # Handle skipped status from LLM - llm_status = llm_output.get("status", "completed") + llm_status = _top_level_or_nested(llm_output, llm_properties, "status", "completed") if llm_status == "skipped": - reason = llm_output.get("reason", "") return self._return_not_applicable_result(reason, self._threshold) - llm_properties = llm_output.get("properties", {}) or {} - - score = float(llm_output.get("score", 0)) + raw_score = _top_level_or_nested(llm_output, llm_properties, "score") + score = _binary_score(raw_score) + if score is None: + raise EvaluationException( + message=f"Invalid score value: {reprlib.repr(raw_score)}. Expected 0 or 1.", + internal_message="Invalid score value.", + blame=ErrorBlame.SYSTEM_ERROR, + category=ErrorCategory.FAILED_EXECUTION, + target=ErrorTarget.TOOL_CALL_SUCCESS_EVALUATOR, + ) success_result = "pass" if score >= 1.0 else "fail" - reason = llm_output.get("reason", "") llm_properties.update(self._get_token_metadata(prompty_output_dict)) return { self._result_key: score, @@ -265,6 +292,61 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[str, float]]: # t ) +def _top_level_or_nested(output: Dict, nested: Dict, key: str, default: Any = None) -> Any: + """Return the judge output's value for ``key``, or the one nested in its properties when it has none.""" + value = output.get(key) + return nested.get(key, default) if value is None else value + + +def _binary_score(value: Any) -> Optional[float]: + """Return a judge's score as 1.0 or 0.0, or None when it is neither.""" + try: + score = float(value) + except (TypeError, ValueError, OverflowError): + return None + if score == 1.0: + return 1.0 + if score == 0.0: + return 0.0 + return None + + +def _has_no_tool_calls(messages) -> bool: + """Return True only for a list of text messages and MCP approvals that holds no tool calls, results or traces. + + A list in any other format may hold tool activity this evaluator does not parse, so the judge assesses it. + """ + if not isinstance(messages, list) or not messages: + return False + for message in messages: + if not isinstance(message, dict) or any(key in message for key in _TOOL_CALL_KEYS): + return False + content = message.get("content") + items = content + if isinstance(content, list): + items = [item for item in content if not (isinstance(item, dict) and item.get("type") in _APPROVAL_TYPES)] + if content and not items: + # Only MCP approval requests or responses, which are not tool calls; a response names its request + # in tool_call_id. + continue + if message.get("role") not in _CONVERSATION_ROLES or "tool_call_id" in message: + return False + if isinstance(content, str): + texts = [content] + elif isinstance(items, list) and all( + isinstance(item, dict) and item.get("type") in _TEXT_CONTENT_TYPES for item in items + ): + texts = [item.get("text") for item in items] + else: + return False + # Text can carry tool calls in this evaluator's own trace format. + if not all( + isinstance(text, str) and not any(marker in text for marker in _TOOL_TRACE_MARKERS) for text in texts + ): + return False + return True + + def _filter_to_used_tools(tool_definitions, msgs_list, logger=None): """Filter the tool definitions to only include those that were actually used in the messages lists.""" try: diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/tool_call_success.prompty b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/tool_call_success.prompty index d7df87a2004d..f9af98623073 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/tool_call_success.prompty +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/tool_call_success.prompty @@ -94,7 +94,7 @@ EXPECTED OUTPUT { "reason": "None of the results indicate an error", "properties": { - "failed_tools": "", + "failed_tools": "" }, "score": 1, "status": "completed" @@ -111,7 +111,7 @@ EXPECTED OUTPUT { "reason": "None of the results indicate an error", "properties": { - "failed_tools": "", + "failed_tools": "" }, "score": 1, "status": "completed" @@ -128,7 +128,7 @@ EXPECTED OUTPUT { "reason": "Although the returned value 7 is not the square root of 4, this is a business mistake in the tool. The tool did not return a result indicating a technical error", "properties": { - "failed_tools": "", + "failed_tools": "" }, "score": 1, "status": "completed" @@ -146,7 +146,7 @@ EXPECTED OUTPUT { "reason": "The tool returned a semicolon separated list of names. Although the description in the definition says it should return comma-separated list , this formatting mistake is a business mistake of the tool, not a technical failure. The tool did not return an error", "properties": { - "failed_tools": "", + "failed_tools": "" }, "score": 1, "status": "completed" @@ -165,7 +165,7 @@ EXPECTED OUTPUT { "reason": "The tool returned empty response which is accepted given that this tool functionality does not include returning data to the caller", "properties": { - "failed_tools": "", + "failed_tools": "" }, "score": 1, "status": "completed" @@ -186,7 +186,7 @@ EXPECTED OUTPUT { "reason": "The tool returned empty response , however , given the tool definition , it should never return empty response because there should be weather info at any given point in time. An empty response here is considered a technical failure. The conclusion is the get_weather_info failed", "properties": { - "failed_tools": "get_weather_info", + "failed_tools": "get_weather_info" }, "score": 0, "status": "completed" @@ -203,7 +203,7 @@ EXPECTED OUTPUT { "reason": "The tool returned a string indicating that it failed", "properties": { - "failed_tools": "get_current_user_Info", + "failed_tools": "get_current_user_Info" }, "score": 0, "status": "completed" @@ -220,7 +220,7 @@ EXPECTED OUTPUT { "reason": "The tool returned an object with empty fields and a string indicating that it failed", "properties": { - "failed_tools": "get_current_user_Info", + "failed_tools": "get_current_user_Info" }, "score": 0, "status": "completed" @@ -237,7 +237,7 @@ EXPECTED OUTPUT { "reason": "The call for GetWeatherInfo returned an object containing single property 'temp' that is an empty string. This means the call to GetWeatherInfo returned empty result while weather info should be available at any time", "properties": { - "failed_tools": "GetWeatherInfo", + "failed_tools": "GetWeatherInfo" }, "score": 0, "status": "completed" @@ -254,7 +254,7 @@ EXPECTED OUTPUT { "reason": "the returned result indicates that the call to get_day_of_week timed out", "properties": { - "failed_tools": "get_day_of_week", + "failed_tools": "get_day_of_week" }, "score": 0, "status": "completed" @@ -272,7 +272,7 @@ EXPECTED OUTPUT { "reason": "null indicates an empty result which cannot be an accepted output of the tool given the tool definition since any given date represents a day of week", "properties": { - "failed_tools": "get_day_of_week", + "failed_tools": "get_day_of_week" }, "score": 0, "status": "completed" @@ -290,7 +290,7 @@ EXPECTED OUTPUT { "reason": "Empty object cannot be an accepted output of the tool given the tool definition since any given date should represent a day of week", "properties": { - "failed_tools": "get_day_of_week", + "failed_tools": "get_day_of_week" }, "score": 0, "status": "completed" @@ -309,7 +309,7 @@ EXPECTED OUTPUT { "reason": "GetWeatherInfo returned an empty response while it should return the weather info and BookTicket returned an error.Both tools failed.", "properties": { - "failed_tools": "GetWeatherInfo,BookTicket", + "failed_tools": "GetWeatherInfo,BookTicket" }, "score": 0, "status": "completed" @@ -328,7 +328,7 @@ EXPECTED OUTPUT { "reason": "Although GetWeatherInfo succeeded, BookTicket returned an error. The final result is failure because one of the tool calls has failed", "properties": { - "failed_tools": "BookTicket", + "failed_tools": "BookTicket" }, "score": 0, "status": "completed" diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_async_token_provider.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_async_token_provider.py index 4509604921c7..71f76db8b7f3 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_async_token_provider.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_async_token_provider.py @@ -4,6 +4,7 @@ import logging import os +import threading from typing import Any, AsyncContextManager, Optional from azure.core.credentials import AccessToken, TokenCredential @@ -27,11 +28,13 @@ def __init__(self, *, base_url: Optional[str] = None, **kwargs: Any) -> None: """Initialize the AsyncAzureTokenProvider.""" self._credential: Optional[TokenCredential] = None self._env_client: Optional[AzureEnvironmentClient] = AzureEnvironmentClient(base_url=base_url, **kwargs) + self._credential_lock = threading.Lock() async def close(self) -> None: - credential, client = self._credential, self._env_client - self._credential = None - self._env_client = None + with self._credential_lock: + credential, client = self._credential, self._env_client + self._credential = None + self._env_client = None try: close_credential = getattr(credential, "close", None) if close_credential is not None: @@ -40,31 +43,58 @@ async def close(self) -> None: if client is not None: await client.close() - async def get_token( - self, - *scopes: str, - claims: Optional[str] = None, - tenant_id: Optional[str] = None, - enable_cae: bool = False, - **kwargs: Any, - ) -> AccessToken: - if self._credential is None: - self._credential = await self._initialize_async(self._env_client) + async def get_credential(self) -> TokenCredential: + """Return the synchronous credential that supplies this provider's tokens, creating it on first use. + + Concurrent first requests all use the first credential created, so they can share its token. + + :return: The credential. + :rtype: ~azure.core.credentials.TokenCredential + """ + credential = self._credential + if credential is None: + created = await self._initialize_async(self._env_client) + with self._credential_lock: + closed = self._env_client is None + # A provider closed while the credential was created keeps no credential. + if self._credential is None and not closed: + self._credential = created + credential = self._credential + if credential is not created: + close_created = getattr(created, "close", None) + if close_created is not None: + close_created() + if closed: + raise EvaluationException( + f"{AsyncAzureTokenProvider.__name__} instance has already been closed.", + target=ErrorTarget.UNKNOWN, + category=ErrorCategory.INVALID_VALUE, + blame=ErrorBlame.USER_ERROR, + ) - if self._credential is None: + if credential is None: raise EvaluationException( f"{self.__class__.__name__} could not determine the credential to use.", target=ErrorTarget.UNKNOWN, category=ErrorCategory.INVALID_VALUE, blame=ErrorBlame.SYSTEM_ERROR, ) + return credential - return self._credential.get_token(*scopes, claims=claims, tenant_id=tenant_id, enable_cae=enable_cae, **kwargs) + async def get_token( + self, + *scopes: str, + claims: Optional[str] = None, + tenant_id: Optional[str] = None, + enable_cae: bool = False, + **kwargs: Any, + ) -> AccessToken: + credential = await self.get_credential() + return credential.get_token(*scopes, claims=claims, tenant_id=tenant_id, enable_cae=enable_cae, **kwargs) async def __aenter__(self) -> "AsyncAzureTokenProvider": try: - if self._credential is None: - self._credential = await self._initialize_async(self._env_client) + await self.get_credential() except BaseException: try: await self.close() diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_token_cache.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_token_cache.py new file mode 100644 index 000000000000..297d50f73d56 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_token_cache.py @@ -0,0 +1,231 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +"""Reuse a caller credential's access tokens across model requests. + +Developer credentials such as ``AzureCliCredential`` start a process for every token request, so evaluations that +requested a token for each model call were slow and, when concurrent, could fail. A token is kept per credential and +scope until shortly before it expires, as azure-core's bearer token policy does. Credentials are keyed by identity +through a weak reference and never kept alive; a credential that cannot be weakly referenced, or a token without a +numeric expiry, is not cached. A token that a service rejected is dropped, and a token whose acquisition began before +that rejection is not stored, so a rejected token never re-enters the cache through a slower request. Requests that +waited on an acquisition whose token was not cached, because of such a rejection or because of its expiry, use that +token instead of each requesting another one. +""" + +import inspect +import math +import threading +import time +import weakref +from typing import Any, Callable, Dict, Optional, Tuple + +from azure.core.exceptions import ClientAuthenticationError + +from azure.ai.evaluation._exceptions import EvaluationException + +# A token is reused until this many seconds before it expires, like azure-core's bearer token policy. +TOKEN_REFRESH_SECONDS = 300 +# How often a request waiting for another request's token checks whether it was abandoned. +TOKEN_WAIT_INTERVAL_SECONDS = 0.05 + + +def _is_fresh(token: Any) -> bool: + expires_on = getattr(token, "expires_on", None) + if isinstance(expires_on, bool) or not isinstance(expires_on, (int, float)): + return False + try: + return ( + isinstance(getattr(token, "token", None), str) + and math.isfinite(expires_on) + and expires_on - time.time() > TOKEN_REFRESH_SECONDS + ) + except OverflowError: + return False + + +def _has_value(token: Any) -> bool: + """Whether a token has a string value to send. Its expiry is not checked: a credential may report none, or 0.""" + return isinstance(getattr(token, "token", None), str) + + +def describe_credential_failure(error: BaseException) -> Optional[Dict[str, Any]]: + """Describe a credential's authentication failure by its message, or return None for any other failure. + + Requests that waited on an authentication failure share it as a ``ClientAuthenticationError``. Any other failure, + such as a token service outage, is not shared, so each waiting request makes its own attempt and raises the + credential's own error type. The description never keeps the error or its traceback. + """ + if isinstance(error, ClientAuthenticationError): + return {"message": str(error) or type(error).__name__} + return None + + +class _CredentialTokens: + """One credential's cached token for one scope, the lock for its token requests, the outcome of its latest token + request and the count of rejections that invalidates acquisitions begun before them.""" + + def __init__(self, reference: Any) -> None: + self.reference = reference + self.token: Any = None + self.lock = threading.Lock() + self.attempts = 0 + self.failure: Optional[Dict[str, Any]] = None + # The latest token request's token when it was not cached, because a rejection kept it out or because of its + # expiry, for the requests that waited on that request. It is never cached. + self.unstored: Any = None + # Incremented when a service rejects a token. An acquisition that began before the latest rejection may + # return the rejected token, so a token is stored only under the generation its acquisition began with. + self.generation = 0 + + +class CredentialTokenCache: + """Share each caller credential's token for a scope across requests until shortly before it expires. + + Concurrent requests that share a synchronous credential wait for one token request. A caller can also ask that + requests which waited on a failed token request share its failure instead of each requesting a token again. A + token a service rejected is discarded; the credential's next token is cached again, even when it is the same + token, while an acquisition that began before the rejection is never stored. Requests that waited on an + acquisition whose token was not cached use that token instead of each requesting another one. + """ + + def __init__(self) -> None: + self._entries: Dict[Tuple[int, str], _CredentialTokens] = {} + # Reentrant because a weak reference callback can run while a lookup holds the lock. + self._guard = threading.RLock() + + def _entry(self, credential: Any, scope: str, create: bool = False) -> Optional[_CredentialTokens]: + key = (id(credential), scope) + with self._guard: + entry = self._entries.get(key) + if entry is not None and entry.reference() is credential: + return entry + if not create: + return None + + def forget(reference: Any) -> None: + with self._guard: + current = self._entries.get(key) + if current is not None and current.reference is reference: + del self._entries[key] + + try: + entry = _CredentialTokens(weakref.ref(credential, forget)) + except TypeError: + return None + self._entries[key] = entry + return entry + + def cached(self, credential: Any, scope: str) -> Any: + entry = self._entry(credential, scope) + with self._guard: + token = entry.token if entry is not None else None + return token if _is_fresh(token) else None + + def generation(self, credential: Any, scope: str) -> int: + """Return the rejection generation to pass to ``store`` for a token acquisition that begins now.""" + entry = self._entry(credential, scope) + with self._guard: + return entry.generation if entry is not None else 0 + + def store(self, credential: Any, scope: str, token: Any, generation: Optional[int] = None) -> bool: + """Keep a fresh token for reuse and return whether it was kept. + + ``generation`` is what ``generation`` returned when the acquisition began. A token acquired before a later + rejection is not stored, because it may be the rejected token; without a generation the token is stored. + """ + entry = self._entry(credential, scope, create=True) if _is_fresh(token) else None + if entry is not None: + with self._guard: + if generation is None or generation == entry.generation: + entry.token = token + return True + return False + + def discard(self, credential: Any, scope: str, token: str) -> None: + """Stop reusing a token a service rejected, and invalidate acquisitions that began before the rejection.""" + entry = self._entry(credential, scope, create=True) + if entry is not None: + with self._guard: + entry.generation += 1 + if entry.token is not None and entry.token.token == token: + entry.token = None + if entry.unstored is not None and entry.unstored.token == token: + entry.unstored = None + + def fetch( + self, + credential: Any, + scope: str, + *, + abandoned: Optional[threading.Event] = None, + deadline: Optional[float] = None, + describe_failure: Optional[Callable[[BaseException], Optional[Dict[str, Any]]]] = None, + failure_type: Callable[..., BaseException] = EvaluationException, + ) -> Any: + """Request a synchronous credential's token once for concurrent requests. + + Returns None, without requesting a token, when the request was abandoned or its monotonic ``deadline`` passed + while it waited for another request's token. With ``describe_failure``, requests that waited on a failed + request raise ``failure_type`` built from its description instead of each requesting a token again; a None + description is not shared. The description must not keep the error, whose traceback references the + credential. Requests that waited on a request whose token was not cached use that token. + """ + entry = self._entry(credential, scope, create=True) + if entry is None: + if (abandoned is not None and abandoned.is_set()) or ( + deadline is not None and time.monotonic() >= deadline + ): + return None + return credential.get_token(scope) + with self._guard: + queued_after = entry.attempts + # Wait in short steps so an abandoned request stops waiting promptly. + while True: + remaining = math.inf if deadline is None else deadline - time.monotonic() + if (abandoned is not None and abandoned.is_set()) or remaining <= 0: + return None + if entry.lock.acquire(timeout=min(remaining, TOKEN_WAIT_INTERVAL_SECONDS)): + break + try: + token = self.cached(credential, scope) + if ( + token is not None + or (abandoned is not None and abandoned.is_set()) + or (deadline is not None and time.monotonic() >= deadline) + ): + return token + with self._guard: + waited = entry.attempts != queued_after + failure = entry.failure if waited and describe_failure is not None else None + unstored = entry.unstored if waited else None + if failure is not None: + raise failure_type(**failure) + if _has_value(unstored): + # Issued for the request this one waited on and not rejected since it was shared, so use it rather + # than request another token. + return unstored + with self._guard: + generation = entry.generation + try: + token = credential.get_token(scope) + except Exception as error: + with self._guard: + entry.attempts += 1 + entry.failure = describe_failure(error) if describe_failure is not None else None + entry.unstored = None + raise + # Decide whether the token is cached and publish the outcome together, so a rejection cannot land between + # them and be undone. + with self._guard: + stored = inspect.isawaitable(token) or self.store(credential, scope, token, generation) + entry.attempts += 1 + entry.failure = None + entry.unstored = token if not stored and _has_value(token) else None + return token + finally: + entry.lock.release() + + +TOKENS = CredentialTokenCache() diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py index 87876d997321..e194b64c24b4 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py @@ -3,17 +3,19 @@ # --------------------------------------------------------- import asyncio +import inspect import re +import threading from logging import Logger from os import PathLike from pathlib import Path -from typing import Any, AsyncGenerator, Awaitable, Dict, Final, List, Mapping, Optional, Sequence, Tuple, Union, cast +from typing import Any, AsyncGenerator, Dict, Final, List, Mapping, Optional, Sequence, Tuple, Union, cast -from openai import AsyncAzureOpenAI, AsyncOpenAI, NotGiven, OpenAIError -from openai.lib.azure import AsyncAzureADTokenProvider +from openai import AsyncAzureOpenAI, AsyncOpenAI, AuthenticationError, NotGiven, OpenAIError from azure.core.credentials import TokenCredential from azure.core.credentials_async import AsyncTokenCredential +from azure.core.exceptions import ClientAuthenticationError from azure.ai.evaluation._exceptions import ErrorTarget from azure.ai.evaluation._constants import DefaultOpenEncoding, TokenScope @@ -41,6 +43,7 @@ from azure.ai.evaluation._constants import DEFAULT_MAX_COMPLETION_TOKENS_REASONING_MODELS from azure.ai.evaluation._legacy._common._logging import get_logger from azure.ai.evaluation._legacy._common._async_token_provider import AsyncAzureTokenProvider +from azure.ai.evaluation._legacy._common._token_cache import TOKENS, describe_credential_failure from azure.ai.evaluation._user_agent import UserAgentSingleton PROMPTY_EXTENSION: Final[str] = ".prompty" @@ -296,6 +299,7 @@ async def __call__( # pylint: disable=docstring-keyword-should-match-keyword-on default_headers = {"User-Agent": UserAgentSingleton().value} api_client: Union[AsyncAzureOpenAI, AsyncOpenAI, AsyncByoProjectResponsesClient] + token_provider: Optional[_ReusedTokenProvider] = None if is_byo: # BYO judge model (connection/deployment): route chat.completions through the Foundry # Responses API, which resolves the connection and auth (API key / managed identity / OAuth2). @@ -311,15 +315,15 @@ async def __call__( # pylint: disable=docstring-keyword-should-match-keyword-on else: connection = Connection.parse_from_config(configuration) if isinstance(connection, AzureOpenAIConnection): + if not connection.api_key: + token_provider = self.get_token_provider(self._token_credential) api_client = AsyncAzureOpenAI( azure_endpoint=connection.azure_endpoint, api_key=connection.api_key, azure_deployment=connection.azure_deployment, api_version=connection.api_version, max_retries=max_retries, - azure_ad_token_provider=( - self.get_token_provider(self._token_credential) if not connection.api_key else None - ), + azure_ad_token_provider=token_provider, default_headers=default_headers, ) elif isinstance(connection, OpenAIConnection): @@ -335,11 +339,17 @@ async def __call__( # pylint: disable=docstring-keyword-should-match-keyword-on f"'{type(connection).__name__}' is not a supported connection type.", target=ErrorTarget.EVAL_RUN ) - response: OpenAIChatResponseType = await self._send_with_retries( - api_client=api_client, - params=params, - timeout=timeout, - ) + try: + response: OpenAIChatResponseType = await self._send_with_retries( + api_client=api_client, + params=params, + timeout=timeout, + ) + except WrappedOpenAIError as error: + if token_provider is not None and isinstance(error.__cause__, AuthenticationError): + # A rejected token is not reused. + token_provider.discard() + raise return await format_llm_response( response=response, @@ -431,18 +441,67 @@ async def _send_with_retries( retry += 1 @staticmethod - def get_token_provider(cred: Union[TokenCredential, AsyncTokenCredential]) -> AsyncAzureADTokenProvider: + def get_token_provider(cred: Union[TokenCredential, AsyncTokenCredential]) -> "_ReusedTokenProvider": """Get the token provider for the prompty. :param Union[TokenCredential, AsyncTokenCredential] cred: The Azure authentication credential. - :return: The token provider if a credential is provided, otherwise None. - :rtype: Optional[AsyncAzureADTokenProvider] + :return: A token provider for the chat client that reuses the credential's token. + :rtype: _ReusedTokenProvider """ + return _ReusedTokenProvider(cred, TokenScope.COGNITIVE_SERVICES_MANAGEMENT) - async def _wrapper() -> str: - token = cred.get_token(TokenScope.COGNITIVE_SERVICES_MANAGEMENT) - if isinstance(token, Awaitable): - token = await token - return token.token - return _wrapper +class _ReusedTokenProvider: + """Supply a credential's token to a chat client, reusing it until shortly before it expires. + + Concurrent requests that share a synchronous credential wait for one token request, on a worker thread so the + event loop is not blocked. The default token provider is represented by the synchronous credential it creates. + The request that made a failed token request raises the credential's error. When that failure was an + authentication failure, requests that waited on it raise a ``ClientAuthenticationError`` with the same message + instead of each requesting a token again; after any other failure, each waiting request makes its own attempt. A + token the service rejected is discarded, and an acquisition that began before the rejection is not stored. + """ + + def __init__(self, credential: Union[TokenCredential, AsyncTokenCredential], scope: str) -> None: + self._credential = credential + self._scope = scope + self._source: Any = credential + self._token: Optional[str] = None + + async def __call__(self) -> str: + credential: Any = self._credential + if isinstance(credential, AsyncAzureTokenProvider): + # Its tokens come from a synchronous credential, so concurrent requests can share one token request. + credential = await credential.get_credential() + self._source = credential + token = TOKENS.cached(credential, self._scope) + if token is None: + # An acquisition that begins before a service rejects a token may return that token; the generation it + # began under lets the cache refuse it. + generation = TOKENS.generation(credential, self._scope) + if inspect.iscoroutinefunction(credential.get_token): + token = await credential.get_token(self._scope) + TOKENS.store(credential, self._scope, token, generation) + else: + abandoned = threading.Event() + try: + token = await asyncio.to_thread( + TOKENS.fetch, + credential, + self._scope, + abandoned=abandoned, + describe_failure=describe_credential_failure, + failure_type=ClientAuthenticationError, + ) + finally: + abandoned.set() + if inspect.isawaitable(token): + token = await token + TOKENS.store(credential, self._scope, token, generation) + self._token = token.token + return self._token + + def discard(self) -> None: + """Stop reusing the token this provider last supplied, which the service rejected.""" + if self._token is not None: + TOKENS.discard(self._source, self._scope, self._token) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index 49c2b863c71e..26e3cfe0c7d7 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -5,12 +5,16 @@ """Offline tests for shared decision-model runners, base selection, and evaluation strategy.""" import asyncio +import gc import hashlib import json import logging import math +import pickle import sys import threading +import time +import weakref from copy import deepcopy from logging.handlers import BufferingHandler from concurrent.futures import ThreadPoolExecutor @@ -19,6 +23,7 @@ import pytest from openai.types.chat import ChatCompletion +from azure.core.credentials import AccessToken from azure.core.pipeline.policies import AsyncRedirectPolicy, AsyncRetryPolicy from azure.core.pipeline.transport import AsyncHttpTransport, HttpResponse from azure.core.exceptions import ServiceRequestError @@ -138,6 +143,42 @@ def run(runner, **overrides): return asyncio.run(runner(**arguments)) +# Only bounds a hung test; no assertion depends on how long anything takes. +WAIT_SECONDS = 10 + + +def watch_token_queue(credential, requests): + """Return an event set once ``requests`` threads have queued for the credential's Decision token request. + + Each request reads the cache's attempt count before it queues, so a token request released after this event + completes while every request is queued, whatever the thread scheduling. + """ + entry = _model_runners._TOKENS._entry(credential, _model_runners._DECISION_TOKEN_SCOPE, create=True) + lock, queued, all_queued = entry.lock, set(), threading.Event() + + class QueueWatchingLock: + def acquire(self, *args, **kwargs): + queued.add(threading.get_ident()) + if len(queued) >= requests: + all_queued.set() + return lock.acquire(*args, **kwargs) + + def release(self): + lock.release() + + entry.lock = QueueWatchingLock() + return all_queued + + +class PicklableCredential: + def __init__(self): + self.requested = 0 + + def get_token(self, *_, **__): + self.requested += 1 + return AccessToken("picklable", int(time.time()) + 3600) + + class TestDecisionModelRunnerConfiguration: @pytest.mark.parametrize( "changes", @@ -541,6 +582,490 @@ def test_unsupported_question_type(self, client): assert not client.calls +class TestDecisionTokenReuse: + """A credential's token serves repeated and concurrent requests until shortly before it expires.""" + + CONFIG = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} + + @staticmethod + def access_token(value, lifetime=3600): + return AccessToken(value, int(time.time()) + lifetime) + + @staticmethod + def authorizations(client): + return [call["headers"]["Authorization"] for call in client.calls] + + @pytest.mark.parametrize("asynchronous", [False, True]) + def test_fresh_token_is_reused_across_requests_and_runners(self, client, asynchronous): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + token = self.access_token("shared-token") + credential = Mock() + credential.get_token = AsyncMock(return_value=token) if asynchronous else Mock(return_value=token) + first, second = DecisionModelRunner(self.CONFIG, credential), DecisionModelRunner(self.CONFIG, credential) + for runner in (first, second, first): + run(runner) + credential.get_token.assert_called_once_with("https://cognitiveservices.azure.com/.default") + assert self.authorizations(client) == ["Bearer shared-token"] * 3 + + @pytest.mark.parametrize("lifetime", [None, 60], ids=["no-expiry", "near-expiry"]) + def test_tokens_without_a_usable_expiry_are_requested_each_time(self, client, lifetime): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + token = SimpleNamespace(token="per-request") if lifetime is None else self.access_token("per-request", lifetime) + credential = Mock(get_token=Mock(return_value=token)) + runner = DecisionModelRunner(self.CONFIG, credential) + run(runner) + run(runner) + assert credential.get_token.call_count == 2 + + def test_concurrent_requests_share_one_synchronous_token_request(self, client): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + release = threading.Event() + requested = [] + + def get_token(*scopes, **_): + requested.append(scopes) + release.wait() + return self.access_token("single-request") + + credential = Mock(get_token=Mock(side_effect=get_token)) + all_queued = watch_token_queue(credential, 6) + runners = [DecisionModelRunner(self.CONFIG, credential) for _ in range(6)] + # Like evaluate(), each runner awaits on its own thread and event loop. + with ThreadPoolExecutor(max_workers=6) as pool: + try: + requests = [pool.submit(run, runner, timeout=2 * WAIT_SECONDS) for runner in runners] + assert all_queued.wait(WAIT_SECONDS) + finally: + release.set() + for request in requests: + request.result(2 * WAIT_SECONDS) + assert len(requested) == 1 + assert self.authorizations(client) == ["Bearer single-request"] * 6 + + @pytest.mark.parametrize("expiry", ["none", "near", "past", "zero"]) + def test_concurrent_requests_share_a_token_that_is_not_cached(self, client, expiry): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + release = threading.Event() + requested = [] + + def get_token(*_, **__): + requested.append(1) + release.wait() + expires_on = {"none": None, "near": time.time() + 120, "past": time.time() - 1, "zero": 0}[expiry] + return SimpleNamespace(token="uncached", expires_on=expires_on) + + credential = Mock(get_token=Mock(side_effect=get_token)) + all_queued = watch_token_queue(credential, 6) + with ThreadPoolExecutor(max_workers=6) as pool: + try: + requests = [ + pool.submit(run, DecisionModelRunner(self.CONFIG, credential), timeout=2 * WAIT_SECONDS) + for _ in range(6) + ] + assert all_queued.wait(WAIT_SECONDS) + finally: + release.set() + for request in requests: + request.result(2 * WAIT_SECONDS) + assert len(requested) == 1 + assert self.authorizations(client) == ["Bearer uncached"] * 6 + assert _model_runners._TOKENS.cached(credential, _model_runners._DECISION_TOKEN_SCOPE) is None + + def test_rejected_token_is_not_reused(self, client): + client.response = FakeResponse(status_code=401, body=b"{}") + tokens = iter([self.access_token("rejected"), self.access_token("renewed")]) + credential = Mock(get_token=Mock(side_effect=lambda *_, **__: next(tokens))) + runner = DecisionModelRunner(self.CONFIG, credential) + with pytest.raises(EvaluationException, match="HTTP 401"): + run(runner) + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + run(runner) + run(runner) + assert credential.get_token.call_count == 2 + assert self.authorizations(client) == ["Bearer rejected", "Bearer renewed", "Bearer renewed"] + + @pytest.mark.parametrize("status", [403, 429, 500]) + def test_a_token_is_kept_after_other_errors(self, client, status): + client.response = FakeResponse(status_code=status, body=b"{}") + credential = Mock(get_token=Mock(return_value=self.access_token("kept"))) + runner = DecisionModelRunner(self.CONFIG, credential, max_retries=0) + with pytest.raises(EvaluationException, match=f"HTTP {status}"): + run(runner) + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + run(runner) + credential.get_token.assert_called_once() + assert self.authorizations(client) == ["Bearer kept"] * 2 + + def test_a_token_the_credential_reissues_after_a_rejection_is_cached_again(self, client): + client.response = FakeResponse(status_code=401, body=b"{}") + credential = Mock(get_token=Mock(return_value=self.access_token("same"))) + runner = DecisionModelRunner(self.CONFIG, credential) + with pytest.raises(EvaluationException, match="HTTP 401"): + run(runner) + assert _model_runners._TOKENS.cached(credential, _model_runners._DECISION_TOKEN_SCOPE) is None + # The service accepts the token now, for example after a role assignment propagated. The credential + # re-issues the same token: it is requested once more, then reused again instead of once per request. + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + for _ in range(3): + run(runner) + assert credential.get_token.call_count == 2 + assert self.authorizations(client) == ["Bearer same"] * 4 + assert _model_runners._TOKENS.cached(credential, _model_runners._DECISION_TOKEN_SCOPE).token == "same" + + def test_an_acquisition_that_began_before_two_rejections_is_not_stored(self): + scope = _model_runners._DECISION_TOKEN_SCOPE + + async def scenario(): + entered, gate = asyncio.Event(), asyncio.Event() + calls = [] + + class AsyncCredential: + async def get_token(self, *_, **__): + calls.append(1) + call = len(calls) + if call == 1: + entered.set() + await gate.wait() + # The delayed first acquisition and the second return token A; later ones return token B. + return TestDecisionTokenReuse.access_token("token-A" if call <= 2 else "token-B") + + credential = AsyncCredential() + runner = DecisionModelRunner(self.CONFIG, credential) + deadline = time.monotonic() + WAIT_SECONDS + pending = asyncio.ensure_future(runner._get_token(deadline)) + await entered.wait() + # Two later acquisitions complete first, and the service rejects each token with a 401. + for expected in ("token-A", "token-B"): + assert await runner._get_token(deadline) == expected + _model_runners._TOKENS.discard(credential, scope, expected) + gate.set() + # The delayed acquisition returns a token the service already rejected; it must not be cached. + assert await pending == "token-A" + assert _model_runners._TOKENS.cached(credential, scope) is None + # The next acquisition is cached again. + assert await runner._get_token(deadline) == "token-B" + assert _model_runners._TOKENS.cached(credential, scope).token == "token-B" + + asyncio.run(scenario()) + + def test_credential_without_weak_references_is_used_without_caching(self, client): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + requested = [] + credential = SimpleNamespace(get_token=lambda *scopes, **_: requested.append(scopes) or self.access_token("t")) + runner = DecisionModelRunner(self.CONFIG, credential) + run(runner) + run(runner) + assert len(requested) == 2 + + def test_unhashable_credential_is_cached_by_identity(self, client): + from dataclasses import dataclass + + @dataclass + class Credential: + requested: int = 0 + + def get_token(self, *_, **__): + self.requested += 1 + return TestDecisionTokenReuse.access_token("by-identity") + + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + credential = Credential() + runner = DecisionModelRunner(self.CONFIG, credential) + run(runner) + run(runner) + assert credential.requested == 1 + + def test_equal_credentials_keep_their_own_tokens(self, client): + class Credential: + def __init__(self, token): + self.token = token + + def __eq__(self, other): + return isinstance(other, Credential) + + def __hash__(self): + return 0 + + def get_token(self, *_, **__): + return TestDecisionTokenReuse.access_token(self.token) + + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + first, second = Credential("first-identity"), Credential("second-identity") + assert first == second + run(DecisionModelRunner(self.CONFIG, first)) + run(DecisionModelRunner(self.CONFIG, second)) + assert self.authorizations(client) == ["Bearer first-identity", "Bearer second-identity"] + + def test_request_whose_deadline_passes_while_waiting_does_not_request_a_token(self, client): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + entered, release = threading.Event(), threading.Event() + requested = [] + + def get_token(*_, **__): + requested.append(1) + entered.set() + release.wait() + raise RuntimeError("Azure CLI is not logged in") + + credential = Mock(get_token=Mock(side_effect=get_token)) + outcomes = {} + + def request(name, timeout): + try: + run(DecisionModelRunner(self.CONFIG, credential), timeout=timeout) + except EvaluationException as error: + outcomes[name] = str(error) + + leader = threading.Thread(target=request, args=("leader", 2 * WAIT_SECONDS)) + waiter = threading.Thread(target=request, args=("waiter", 0.3)) + try: + leader.start() + assert entered.wait(WAIT_SECONDS) + # The first request holds the credential's token request until the second request's deadline has passed. + waiter.start() + waiter.join(WAIT_SECONDS) + waiter_finished = not waiter.is_alive() + finally: + release.set() + leader.join(WAIT_SECONDS) + assert waiter_finished and not leader.is_alive() + assert len(requested) == 1 + assert "timed out" in outcomes["waiter"] + assert "Could not acquire a token" in outcomes["leader"] + + def test_requests_waiting_on_a_failed_token_request_share_its_failure(self, client): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + entered, release = threading.Event(), threading.Event() + requested = [] + + def get_token(*_, **__): + requested.append(1) + entered.set() + release.wait() + raise EvaluationException( + message="Service-owned Decision credential acquisition failed (RuntimeError, status unknown).", + blame=ErrorBlame.SYSTEM_ERROR, + category=ErrorCategory.FAILED_EXECUTION, + target=ErrorTarget.EVALUATE, + ) + + credential = Mock(get_token=Mock(side_effect=get_token)) + all_queued = watch_token_queue(credential, 4) + errors = {} + + def request(name): + try: + run(DecisionModelRunner(self.CONFIG, credential), timeout=2 * WAIT_SECONDS) + except EvaluationException as error: + errors[name] = error + + leader = threading.Thread(target=request, args=("leader",)) + waiters = [threading.Thread(target=request, args=(f"waiter-{index}",)) for index in range(3)] + try: + leader.start() + assert entered.wait(WAIT_SECONDS) + for waiter in waiters: + waiter.start() + assert all_queued.wait(WAIT_SECONDS) + finally: + release.set() + for thread in [leader, *waiters]: + thread.join(WAIT_SECONDS) + # The queued requests fail with the same typed error instead of each starting another request. + assert len(requested) == 1 and len(errors) == 4 + assert all( + error.blame == ErrorBlame.SYSTEM_ERROR and "acquisition failed" in str(error) for error in errors.values() + ) + credential.get_token.side_effect = lambda *_, **__: self.access_token("recovered") + run(DecisionModelRunner(self.CONFIG, credential)) + assert len(requested) == 1 and self.authorizations(client) == ["Bearer recovered"] + + def test_requests_waiting_on_a_token_kept_out_of_the_cache_use_it(self, client): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + entered, release = threading.Event(), threading.Event() + requested = [] + + def get_token(*_, **__): + requested.append(1) + entered.set() + release.wait() + return self.access_token("issued") + + credential = Mock(get_token=Mock(side_effect=get_token)) + all_queued = watch_token_queue(credential, 4) + scope = _model_runners._DECISION_TOKEN_SCOPE + with ThreadPoolExecutor(max_workers=4) as pool: + try: + requests = [ + pool.submit(run, DecisionModelRunner(self.CONFIG, credential), timeout=2 * WAIT_SECONDS) + for _ in range(4) + ] + assert entered.wait(WAIT_SECONDS) and all_queued.wait(WAIT_SECONDS) + # Another request's 401 lands while the token is requested, so the token is not cached. + _model_runners._TOKENS.discard(credential, scope, "rejected-earlier") + finally: + release.set() + for request in requests: + request.result(2 * WAIT_SECONDS) + assert len(requested) == 1 + assert self.authorizations(client) == ["Bearer issued"] * 4 + assert _model_runners._TOKENS.cached(credential, scope) is None + # A request that starts afterwards requests its own token, which is cached again. + run(DecisionModelRunner(self.CONFIG, credential)) + assert len(requested) == 2 + assert _model_runners._TOKENS.cached(credential, scope).token == "issued" + + def test_a_token_rejected_before_its_request_resumes_is_not_stored_again(self, client, monkeypatch): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + credential = Mock(get_token=Mock(return_value=self.access_token("shared"))) + fetch = _model_runners._TOKENS.fetch + scope = _model_runners._DECISION_TOKEN_SCOPE + + def fetch_then_reject(*args, **kwargs): + token = fetch(*args, **kwargs) + # Another request's 401 discards the token before this request's event loop resumes. + _model_runners._TOKENS.discard(credential, scope, token.token) + return token + + monkeypatch.setattr(_model_runners._TOKENS, "fetch", fetch_then_reject) + run(DecisionModelRunner(self.CONFIG, credential)) + assert _model_runners._TOKENS.cached(credential, scope) is None + + def test_failed_token_request_does_not_keep_the_credential_alive(self, client): + class Credential: + def get_token(self, *_, **__): + raise RuntimeError("Azure CLI is not logged in") + + credential = Credential() + with pytest.raises(EvaluationException, match="Could not acquire a token"): + run(DecisionModelRunner(self.CONFIG, credential)) + reference = weakref.ref(credential) + del credential + gc.collect() + assert reference() is None + + def test_cancelled_request_stops_waiting_for_another_token_request(self, client): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + entered, release = threading.Event(), threading.Event() + requested = [] + + def get_token(*_, **__): + requested.append(1) + entered.set() + release.wait() + return self.access_token("late") + + credential = Mock(get_token=Mock(side_effect=get_token)) + all_queued = watch_token_queue(credential, 2) + cancelled = threading.Event() + outcomes = {} + + def waiter(): + try: + run( + DecisionModelRunner(self.CONFIG, credential, cancellation_requested=cancelled.is_set), + timeout=2 * WAIT_SECONDS, + ) + except EvaluationException as error: + outcomes["waiter"] = str(error) + + leader = threading.Thread( + target=lambda: run(DecisionModelRunner(self.CONFIG, credential), timeout=2 * WAIT_SECONDS) + ) + waiting = threading.Thread(target=waiter) + try: + leader.start() + assert entered.wait(WAIT_SECONDS) + waiting.start() + # Cancel only once the second request waits for the first request's token. + assert all_queued.wait(WAIT_SECONDS) + cancelled.set() + # The first request's token request stays blocked until ``finally``, so the cancelled request can only + # return by giving up its wait. + waiting.join(WAIT_SECONDS) + still_waiting = waiting.is_alive() + finally: + release.set() + leader.join(WAIT_SECONDS) + waiting.join(WAIT_SECONDS) + assert not still_waiting and "cancelled" in outcomes["waiter"] + assert len(requested) == 1 + + def test_cache_does_not_keep_the_credential_alive(self, client): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + token = self.access_token("caller-owned") + + class Credential: + def get_token(self, *_, **__): + return token + + credential = Credential() + run(DecisionModelRunner(self.CONFIG, credential)) + assert _model_runners._TOKENS.cached(credential, _model_runners._DECISION_TOKEN_SCOPE) is token + reference = weakref.ref(credential) + del credential + gc.collect() + assert reference() is None + + def test_a_runner_still_pickles_after_its_token_is_cached(self, client): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + credential = PicklableCredential() + runner = DecisionModelRunner(self.CONFIG, credential) + run(runner) + copy = pickle.loads(pickle.dumps(runner)) + # The copy's credential is another object, so it requests its own token once and then reuses it. + run(copy) + run(copy) + run(runner) + assert credential.requested == 1 + assert copy._credential.requested == 2 + assert self.authorizations(client) == ["Bearer picklable"] * 4 + + @pytest.mark.parametrize("asynchronous", [False, True]) + @pytest.mark.parametrize( + "token", + [None, SimpleNamespace(token=None, expires_on=None), SimpleNamespace(token=123, expires_on=None)], + ids=["none", "no-token", "non-string-token"], + ) + def test_a_credential_that_returns_no_usable_token_is_a_user_error(self, client, token, asynchronous): + credential = Mock() + credential.get_token = AsyncMock(return_value=token) if asynchronous else Mock(return_value=token) + with pytest.raises(EvaluationException, match="Could not acquire a token") as raised: + run(DecisionModelRunner(self.CONFIG, credential)) + assert raised.value.blame == ErrorBlame.USER_ERROR + assert not client.calls + + def test_a_token_wait_ends_at_the_deadline_without_requesting_a_token(self): + credential = Mock(get_token=Mock(return_value=self.access_token("unused"))) + scope = _model_runners._DECISION_TOKEN_SCOPE + entry = _model_runners._TOKENS._entry(credential, scope, create=True) + runner = DecisionModelRunner(self.CONFIG, credential) + # Another request holds the credential's token request throughout. + assert entry.lock.acquire(timeout=WAIT_SECONDS) + try: + assert _model_runners._TOKENS.fetch(credential, scope, deadline=time.monotonic() + 0.1) is None + assert asyncio.run(runner._get_token(time.monotonic() + 0.1)) is None + finally: + entry.lock.release() + credential.get_token.assert_not_called() + + def test_a_credential_without_weak_references_is_not_asked_after_the_deadline(self): + requested = [] + credential = SimpleNamespace(get_token=lambda *scopes, **_: requested.append(scopes)) + scope = _model_runners._DECISION_TOKEN_SCOPE + + assert _model_runners._TOKENS.fetch(credential, scope, deadline=time.monotonic() - 1) is None + assert requested == [] + + def test_a_request_whose_token_wait_reached_the_deadline_times_out(self, client, monkeypatch): + runner = DecisionModelRunner(self.CONFIG, Mock(get_token=Mock())) + monkeypatch.setattr(runner, "_get_token", AsyncMock(return_value=None)) + with pytest.raises(EvaluationException, match="timed out") as raised: + run(runner) + assert raised.value.blame == ErrorBlame.SYSTEM_ERROR + assert not client.calls + + class TestAnswerValidation: def test_noul(self): assert validate_noul_answer({"type": "noul", "noul": 0.25}) == 0.25 diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_token_reuse.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_token_reuse.py new file mode 100644 index 000000000000..d67f5f225670 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_token_reuse.py @@ -0,0 +1,647 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +"""The chat flow's token provider reuses a caller credential's token, kept apart from Decision tokens.""" + +import asyncio +import gc +import threading +import time +import weakref +from pathlib import Path +from types import SimpleNamespace + +import httpx +import pytest +from azure.core.credentials import AccessToken +from azure.core.exceptions import ClientAuthenticationError +from openai import AuthenticationError, InternalServerError + +import azure.ai.evaluation._evaluators._coherence as coherence +from azure.ai.evaluation._common.utils import construct_prompty_model_config, validate_model_config +from azure.ai.evaluation._constants import TokenScope +from azure.ai.evaluation._exceptions import EvaluationException +from azure.ai.evaluation._legacy._common._async_token_provider import AsyncAzureTokenProvider +from azure.ai.evaluation._legacy._common._token_cache import TOKENS +from azure.ai.evaluation._legacy.prompty import AsyncPrompty +from azure.ai.evaluation._legacy.prompty._exceptions import WrappedOpenAIError + +CHAT_SCOPE = TokenScope.COGNITIVE_SERVICES_MANAGEMENT +DECISION_SCOPE = "https://cognitiveservices.azure.com/.default" +COHERENCE_PROMPTY = Path(coherence.__file__).parent / "coherence.prompty" +CHAT_MODEL = construct_prompty_model_config( + validate_model_config({"azure_endpoint": "https://example.openai.azure.com", "azure_deployment": "judge"}), + "2024-10-21", + "test-agent", +) +# Only bounds a hung test; no assertion depends on how long anything takes. +WAIT_SECONDS = 10 + + +def access_token(value): + return AccessToken(value, int(time.time()) + 3600) + + +def request_token(credential): + return asyncio.run(AsyncPrompty.get_token_provider(credential)()) + + +def join_all(threads): + for thread in threads: + thread.join(WAIT_SECONDS) + assert not any(thread.is_alive() for thread in threads) + + +def watch_token_queue(credential, requests): + """Return an event set once ``requests`` threads have queued for the credential's chat token request. + + Each request reads the cache's attempt count before it queues, so a token request released after this event + completes while every request is queued, whatever the thread scheduling. + """ + entry = TOKENS._entry(credential, CHAT_SCOPE, create=True) + lock, queued, all_queued = entry.lock, set(), threading.Event() + + class QueueWatchingLock: + def acquire(self, *args, **kwargs): + queued.add(threading.get_ident()) + if len(queued) >= requests: + all_queued.set() + return lock.acquire(*args, **kwargs) + + def release(self): + lock.release() + + entry.lock = QueueWatchingLock() + return all_queued + + +class Credential: + """A synchronous credential that records each token request and answers from a queue of results. + + With ``gate``, each token request waits until the test sets the event; tests set it in ``finally``. + """ + + def __init__(self, *results, gate=None): + self.scopes = [] + self._results = list(results) + self._gate = gate + self._lock = threading.Lock() + + def get_token(self, *scopes, **_): + with self._lock: + self.scopes.append(scopes) + result = self._results.pop(0) if len(self._results) > 1 else self._results[0] + if self._gate is not None: + self._gate.wait() + if isinstance(result, Exception): + raise result + return result + + +@pytest.mark.unittest +class TestChatTokenReuse: + def test_a_fresh_token_is_reused_across_requests_and_flows(self): + credential = Credential(access_token("chat")) + + assert [request_token(credential) for _ in range(3)] == ["chat"] * 3 + assert asyncio.run(AsyncPrompty.get_token_provider(credential)()) == "chat" + assert credential.scopes == [(CHAT_SCOPE,)] + + def test_concurrent_requests_wait_for_one_token_request(self): + release = threading.Event() + credential = Credential(access_token("shared"), gate=release) + all_queued = watch_token_queue(credential, 8) + tokens = [] + threads = [threading.Thread(target=lambda: tokens.append(request_token(credential))) for _ in range(8)] + try: + for thread in threads: + thread.start() + assert all_queued.wait(WAIT_SECONDS) + finally: + release.set() + join_all(threads) + + assert tokens == ["shared"] * 8 + assert credential.scopes == [(CHAT_SCOPE,)] + + @pytest.mark.parametrize("expiry", ["none", "near", "past", "zero"]) + def test_concurrent_requests_share_a_token_that_is_not_cached(self, expiry): + expires_on = {"none": None, "near": time.time() + 120, "past": time.time() - 1, "zero": 0}[expiry] + issued = SimpleNamespace(token="uncached", expires_on=expires_on) + release = threading.Event() + credential = Credential(issued, gate=release) + all_queued = watch_token_queue(credential, 6) + tokens = [] + threads = [threading.Thread(target=lambda: tokens.append(request_token(credential))) for _ in range(6)] + try: + for thread in threads: + thread.start() + assert all_queued.wait(WAIT_SECONDS) + finally: + release.set() + join_all(threads) + + assert tokens == ["uncached"] * 6 + assert len(credential.scopes) == 1 + assert TOKENS.cached(credential, CHAT_SCOPE) is None + # A request that did not wait on that token request makes its own. + assert request_token(credential) == "uncached" + assert len(credential.scopes) == 2 + + def test_a_token_without_a_value_is_not_given_to_waiting_requests(self): + release = threading.Event() + credential = Credential(SimpleNamespace(token=None, expires_on=None), gate=release) + all_queued = watch_token_queue(credential, 3) + outcomes = [] + threads = [threading.Thread(target=lambda: outcomes.append(request_token(credential))) for _ in range(3)] + try: + for thread in threads: + thread.start() + assert all_queued.wait(WAIT_SECONDS) + finally: + release.set() + join_all(threads) + + # Each request asks the credential itself and returns its value, as before. + assert outcomes == [None] * 3 + assert len(credential.scopes) == 3 + + def test_waiting_for_another_requests_token_does_not_block_the_event_loop(self): + entered, release = threading.Event(), threading.Event() + + class SlowCredential(Credential): + def get_token(self, *scopes, **kwargs): + entered.set() + return super().get_token(*scopes, **kwargs) + + credential = SlowCredential(access_token("shared"), gate=release) + all_queued = watch_token_queue(credential, 2) + leader = threading.Thread(target=lambda: request_token(credential)) + # Releases the token request if waiting blocked the loop, so a regression fails instead of hanging. + watchdog = threading.Timer(WAIT_SECONDS, release.set) + + async def scenario(): + waiting = asyncio.ensure_future(AsyncPrompty.get_token_provider(credential)()) + assert await asyncio.to_thread(all_queued.wait, WAIT_SECONDS) + # The loop ran this coroutine while the request waits for the first request's token. + assert not waiting.done() and not release.is_set() + release.set() + return await waiting + + try: + watchdog.start() + leader.start() + assert entered.wait(WAIT_SECONDS) + assert asyncio.run(scenario()) == "shared" + finally: + release.set() + watchdog.cancel() + join_all([leader]) + assert len(credential.scopes) == 1 + + def test_chat_and_decision_tokens_are_kept_apart(self): + credential = Credential(access_token("chat")) + TOKENS.store(credential, DECISION_SCOPE, access_token("decision")) + + assert request_token(credential) == "chat" + assert credential.scopes == [(CHAT_SCOPE,)] + assert TOKENS.cached(credential, DECISION_SCOPE).token == "decision" + + def test_a_failed_token_request_raises_the_credential_error_and_is_retried(self): + credential = Credential(ClientAuthenticationError("Please run az login."), access_token("recovered")) + + with pytest.raises(ClientAuthenticationError, match="az login"): + request_token(credential) + assert request_token(credential) == "recovered" + assert len(credential.scopes) == 2 + + def test_requests_queued_behind_a_failed_token_request_share_its_failure(self): + release = threading.Event() + credential = Credential( + ClientAuthenticationError("Please run az login."), access_token("recovered"), gate=release + ) + all_queued = watch_token_queue(credential, 4) + outcomes = [] + + def request(): + try: + outcomes.append(request_token(credential)) + except ClientAuthenticationError as error: + outcomes.append(str(error)) + + threads = [threading.Thread(target=request) for _ in range(4)] + try: + for thread in threads: + thread.start() + assert all_queued.wait(WAIT_SECONDS) + finally: + release.set() + join_all(threads) + + assert outcomes == ["Please run az login."] * 4 + assert len(credential.scopes) == 1 + # A request made after the failure tries again. + assert request_token(credential) == "recovered" + + def test_other_failures_are_not_shared_so_each_request_keeps_the_error_type(self): + class TokenServiceOutage(Exception): + pass + + release = threading.Event() + credential = Credential(TokenServiceOutage("The token service is unavailable."), gate=release) + all_queued = watch_token_queue(credential, 3) + outcomes = [] + + def request(): + try: + outcomes.append(request_token(credential)) + except Exception as error: # pylint: disable=broad-except + outcomes.append(type(error).__name__) + + threads = [threading.Thread(target=request) for _ in range(3)] + try: + for thread in threads: + thread.start() + assert all_queued.wait(WAIT_SECONDS) + finally: + release.set() + join_all(threads) + + assert outcomes == ["TokenServiceOutage"] * 3 + assert len(credential.scopes) == 3 + + def test_requests_waiting_on_a_token_kept_out_of_the_cache_use_it(self): + entered, release = threading.Event(), threading.Event() + + class SlowCredential(Credential): + def get_token(self, *scopes, **kwargs): + entered.set() + release.wait() + return super().get_token(*scopes, **kwargs) + + credential = SlowCredential(access_token("issued")) + all_queued = watch_token_queue(credential, 4) + tokens = [] + threads = [threading.Thread(target=lambda: tokens.append(request_token(credential))) for _ in range(4)] + try: + for thread in threads: + thread.start() + assert entered.wait(WAIT_SECONDS) and all_queued.wait(WAIT_SECONDS) + # Another request's 401 lands while the token is requested, so the token is not cached. + TOKENS.discard(credential, CHAT_SCOPE, "rejected-earlier") + finally: + release.set() + join_all(threads) + + assert tokens == ["issued"] * 4 + assert len(credential.scopes) == 1 + assert TOKENS.cached(credential, CHAT_SCOPE) is None + # A request that starts afterwards requests its own token, which is cached again. + assert request_token(credential) == "issued" + assert len(credential.scopes) == 2 + assert TOKENS.cached(credential, CHAT_SCOPE).token == "issued" + + def test_a_rejected_token_is_not_given_to_waiting_requests(self): + credential = Credential(access_token("unused")) + entry = TOKENS._entry(credential, CHAT_SCOPE, create=True) + entry.unstored = access_token("issued") + + TOKENS.discard(credential, CHAT_SCOPE, "other") + assert entry.unstored.token == "issued" + TOKENS.discard(credential, CHAT_SCOPE, "issued") + assert entry.unstored is None + + def test_a_rejection_cannot_land_between_caching_and_the_hand_off_to_waiting_requests(self, monkeypatch): + credential = Credential(access_token("issued")) + store = TOKENS.store + guard_free = [] + + def probe_guard(): + # A rejection from another request needs this lock. + acquired = TOKENS._guard.acquire(blocking=False) + if acquired: + TOKENS._guard.release() + guard_free.append(acquired) + + def store_then_probe(*args, **kwargs): + # A rejection while the token was requested keeps it out of the cache for waiting requests. + TOKENS.discard(credential, CHAT_SCOPE, "rejected-earlier") + stored = store(*args, **kwargs) + probe = threading.Thread(target=probe_guard) + probe.start() + join_all([probe]) + return stored + + monkeypatch.setattr(TOKENS, "store", store_then_probe) + assert request_token(credential) == "issued" + + # The decision to cache and the hand-off happen while the lock is held, so no rejection lands between them. + assert guard_free == [False] + assert TOKENS._entry(credential, CHAT_SCOPE).unstored.token == "issued" + # A rejection afterwards still reaches the handed-off token. + TOKENS.discard(credential, CHAT_SCOPE, "issued") + assert TOKENS._entry(credential, CHAT_SCOPE).unstored is None + + def test_requests_waiting_on_a_failed_token_request_do_not_get_an_older_token(self): + class TokenServiceOutage(Exception): + pass + + entered, first_release, second_release = threading.Event(), threading.Event(), threading.Event() + calls = [] + + class StagedCredential(Credential): + def get_token(self, *scopes, **kwargs): + calls.append(1) + if len(calls) == 1: + entered.set() + first_release.wait() + elif len(calls) == 2: + second_release.wait() + return super().get_token(*scopes, **kwargs) + + credential = StagedCredential( + access_token("issued"), TokenServiceOutage("The token service is unavailable."), access_token("recovered") + ) + results = {} + + def request(name): + try: + results[name] = request_token(credential) + except Exception as error: # pylint: disable=broad-except + results[name] = type(error).__name__ + + # A rejection while the first token is requested keeps that token out of the cache. + first = threading.Thread(target=request, args=("first",)) + try: + first.start() + assert entered.wait(WAIT_SECONDS) + TOKENS.discard(credential, CHAT_SCOPE, "rejected-earlier") + finally: + first_release.set() + join_all([first]) + assert results["first"] == "issued" and TOKENS.cached(credential, CHAT_SCOPE) is None + + # The next token request fails; the requests waiting on it make their own attempt. + all_queued = watch_token_queue(credential, 3) + names = ["second", "waiter-1", "waiter-2"] + threads = [threading.Thread(target=request, args=(name,)) for name in names] + try: + for thread in threads: + thread.start() + assert all_queued.wait(WAIT_SECONDS) + finally: + second_release.set() + join_all(threads) + + assert sorted(results[name] for name in names) == ["TokenServiceOutage", "recovered", "recovered"] + assert len(calls) == 3 + + def test_a_token_without_an_expiry_is_requested_each_time(self): + credential = Credential(SimpleNamespace(token="no-expiry", expires_on=None)) + + assert [request_token(credential) for _ in range(2)] == ["no-expiry"] * 2 + assert len(credential.scopes) == 2 + + def test_an_asynchronous_credentials_token_is_reused(self): + class AsyncCredential: + def __init__(self): + self.calls = 0 + + async def get_token(self, *_, **__): + self.calls += 1 + return access_token("async") + + credential = AsyncCredential() + + assert [request_token(credential) for _ in range(3)] == ["async"] * 3 + assert credential.calls == 1 + + def test_the_cache_does_not_keep_the_credential_alive(self): + credential = Credential(access_token("caller-owned")) + assert request_token(credential) == "caller-owned" + assert TOKENS.cached(credential, CHAT_SCOPE) is not None + + reference = weakref.ref(credential) + del credential + gc.collect() + assert reference() is None + + @pytest.mark.parametrize("default_provider", [False, True], ids=["caller credential", "default provider"]) + @pytest.mark.parametrize( + "error_type, status, tokens_sent", + [(AuthenticationError, 401, ["rejected", "replacement"]), (InternalServerError, 500, ["rejected", "rejected"])], + ) + def test_only_a_token_rejected_with_401_is_requested_again( + self, monkeypatch, error_type, status, tokens_sent, default_provider + ): + credential = Credential(access_token("rejected"), access_token("replacement")) + if default_provider: + provider = AsyncAzureTokenProvider() + provider._credential = credential + token_credential = provider + else: + token_credential = credential + flow = AsyncPrompty.load(source=COHERENCE_PROMPTY, model=CHAT_MODEL, token_credential=token_credential) + sent = [] + + async def send(self, *, api_client, params, timeout): + sent.append(await api_client._azure_ad_token_provider()) + response = httpx.Response(status, request=httpx.Request("POST", "https://example.openai.azure.com")) + error = error_type("request failed", response=response, body=None) + raise WrappedOpenAIError(error=error) from error + + monkeypatch.setattr(AsyncPrompty, "_send_with_retries", send) + for _ in range(2): + with pytest.raises(WrappedOpenAIError): + asyncio.run(flow(query="When does the museum open?", response="At noon.")) + + assert sent == tokens_sent + assert len(credential.scopes) == len(set(tokens_sent)) + + def test_the_default_provider_reuses_its_token(self): + provider = AsyncAzureTokenProvider() + inner = Credential(access_token("default")) + provider._credential = inner + + assert [request_token(provider) for _ in range(3)] == ["default"] * 3 + assert inner.scopes == [(CHAT_SCOPE,)] + asyncio.run(provider.close()) + + def test_concurrent_requests_share_the_default_providers_token_request(self): + release = threading.Event() + provider = AsyncAzureTokenProvider() + inner = Credential(access_token("default"), gate=release) + provider._credential = inner + all_queued = watch_token_queue(inner, 6) + tokens = [] + threads = [threading.Thread(target=lambda: tokens.append(request_token(provider))) for _ in range(6)] + try: + for thread in threads: + thread.start() + assert all_queued.wait(WAIT_SECONDS) + finally: + release.set() + join_all(threads) + + assert tokens == ["default"] * 6 + assert inner.scopes == [(CHAT_SCOPE,)] + asyncio.run(provider.close()) + + def test_the_default_provider_creates_its_credential_once(self, monkeypatch): + inner = Credential(access_token("created")) + created = [] + + async def initialize(client): + created.append(client) + return inner + + monkeypatch.setattr(AsyncAzureTokenProvider, "_initialize_async", staticmethod(initialize)) + provider = AsyncAzureTokenProvider() + + assert [request_token(provider) for _ in range(3)] == ["created"] * 3 + assert len(created) == 1 and inner.scopes == [(CHAT_SCOPE,)] + # Its own get_token still asks the credential it created. + assert asyncio.run(provider.get_token(CHAT_SCOPE)).token == "created" + assert len(inner.scopes) == 2 + asyncio.run(provider.close()) + + def test_concurrent_first_requests_share_one_default_credential_and_token_request(self, monkeypatch): + requests = 4 + barrier = threading.Barrier(requests) + created, closed = [], [] + + class ClosableCredential(Credential): + def close(self): + closed.append(self) + + async def initialize(client): + # Every first request creates a credential at the same time. + await asyncio.to_thread(barrier.wait, WAIT_SECONDS) + credential = ClosableCredential(access_token("default")) + created.append(credential) + return credential + + monkeypatch.setattr(AsyncAzureTokenProvider, "_initialize_async", staticmethod(initialize)) + provider = AsyncAzureTokenProvider() + tokens = [] + threads = [threading.Thread(target=lambda: tokens.append(request_token(provider))) for _ in range(requests)] + for thread in threads: + thread.start() + join_all(threads) + + assert tokens == ["default"] * requests + # The first credential created is kept and the others are closed, so one token request serves every row. + assert len(created) == requests and len(closed) == requests - 1 + assert sum(len(credential.scopes) for credential in created) == 1 + asyncio.run(provider.close()) + + def test_a_credential_created_while_the_default_provider_closes_is_closed(self, monkeypatch): + closed = [] + + class ClosableCredential(Credential): + def close(self): + closed.append(self) + + created = ClosableCredential(access_token("late")) + + async def scenario(): + entered, gate = asyncio.Event(), asyncio.Event() + + async def initialize(client): + entered.set() + await gate.wait() + return created + + monkeypatch.setattr(AsyncAzureTokenProvider, "_initialize_async", staticmethod(initialize)) + provider = AsyncAzureTokenProvider() + pending = asyncio.ensure_future(provider.get_credential()) + await entered.wait() + await provider.close() + gate.set() + with pytest.raises(EvaluationException, match="already been closed"): + await pending + assert provider._credential is None + + asyncio.run(scenario()) + assert closed == [created] + + def test_a_token_rejected_while_another_request_awaits_it_is_not_stored_again(self): + async def scenario(): + entered, gate = asyncio.Event(), asyncio.Event() + calls = [] + + class AsyncCredential: + async def get_token(self, *_, **__): + calls.append(1) + if len(calls) == 1: + entered.set() + await gate.wait() + return access_token("rejected") + + credential = AsyncCredential() + waiting = AsyncPrompty.get_token_provider(credential) + rejected = AsyncPrompty.get_token_provider(credential) + pending = asyncio.ensure_future(waiting()) + # The first request misses the cache and awaits its token. + await entered.wait() + assert await rejected() == "rejected" + # The service rejects that token with a 401 before the first request resumes. + rejected.discard() + gate.set() + assert await pending == "rejected" + assert TOKENS.cached(credential, CHAT_SCOPE) is None + # A different token from the credential is cached again. + TOKENS.store(credential, CHAT_SCOPE, access_token("new")) + assert TOKENS.cached(credential, CHAT_SCOPE).token == "new" + + asyncio.run(scenario()) + + def test_a_token_the_credential_reissues_after_a_rejection_is_cached_again(self): + credential = Credential(access_token("same")) + + async def reject_once(): + provider = AsyncPrompty.get_token_provider(credential) + assert await provider() == "same" + # The service rejects the token with a 401, for example while a role assignment still propagates. + provider.discard() + + asyncio.run(reject_once()) + assert TOKENS.cached(credential, CHAT_SCOPE) is None + # The credential re-issues the same token: it is requested once more, then reused again. + assert [request_token(credential) for _ in range(3)] == ["same"] * 3 + assert len(credential.scopes) == 2 + assert TOKENS.cached(credential, CHAT_SCOPE).token == "same" + + def test_an_acquisition_that_began_before_two_rejections_is_not_stored(self): + async def scenario(): + entered, gate = asyncio.Event(), asyncio.Event() + calls = [] + + class AsyncCredential: + async def get_token(self, *_, **__): + calls.append(1) + call = len(calls) + if call == 1: + entered.set() + await gate.wait() + # The delayed first acquisition and the second return token A; later ones return token B. + return access_token("token-A" if call <= 2 else "token-B") + + credential = AsyncCredential() + pending = asyncio.ensure_future(AsyncPrompty.get_token_provider(credential)()) + await entered.wait() + # Two later acquisitions complete first, and the service rejects each token with a 401. + for expected in ("token-A", "token-B"): + provider = AsyncPrompty.get_token_provider(credential) + assert await provider() == expected + provider.discard() + gate.set() + # The delayed acquisition returns a token the service already rejected; it must not be cached. + assert await pending == "token-A" + assert TOKENS.cached(credential, CHAT_SCOPE) is None + # The next acquisition is cached again. + assert await AsyncPrompty.get_token_provider(credential)() == "token-B" + assert TOKENS.cached(credential, CHAT_SCOPE).token == "token-B" + + asyncio.run(scenario()) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py new file mode 100644 index 000000000000..5cb144c81ee3 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py @@ -0,0 +1,387 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +"""Tool Call Success reads the judge's score where judge models put it, and skips responses without tool calls.""" + +import json +import math +import re +from pathlib import Path +from unittest.mock import AsyncMock + +import pandas as pd +import pytest + +from azure.ai.evaluation import evaluate +from azure.ai.evaluation._evaluators._tool_call_success import _tool_call_success as module +from azure.ai.evaluation._evaluators._tool_call_success._tool_call_success import ( + _ToolCallSuccessEvaluator, + _has_no_tool_calls, +) +from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException + +CHAT_CONFIG = { + "azure_endpoint": "https://example.openai.azure.com", + "azure_deployment": "judge", + "api_key": "test-key", +} +DECISION_CONFIG = { + "type": "decision", + "azure_endpoint": "https://example.services.ai.azure.com", + "azure_deployment": "Decision-1", + "api_key": "test-key", +} +PROMPTY = Path(module.__file__).parent / "tool_call_success.prompty" +TOOL_CALL_RESPONSE = [ + { + "role": "assistant", + "content": [ + {"type": "tool_call", "tool_call_id": "call_1", "name": "fetch_weather", "arguments": {"city": "Paris"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": [{"type": "tool_result", "tool_result": {"sky": "sunny"}}]}, + {"role": "assistant", "content": [{"type": "text", "text": "It is sunny in Paris."}]}, +] +TEXT_RESPONSE = [ + {"role": "user", "content": [{"type": "text", "text": "What is the weather in Paris?"}]}, + {"role": "assistant", "content": [{"type": "output_text", "text": "I can't check the weather."}]}, +] +JUDGE_USAGE = {"input_token_count": 11, "output_token_count": 7, "total_token_count": 18} + + +def judge(llm_output): + """Return a chat evaluator whose judge model answers with ``llm_output``.""" + evaluator = _ToolCallSuccessEvaluator(model_config=dict(CHAT_CONFIG)) + evaluator._flow = AsyncMock(return_value={"llm_output": llm_output, **JUDGE_USAGE}) + return evaluator + + +def evaluate_with(llm_output): + return judge(llm_output)(response=TOOL_CALL_RESPONSE) + + +def prompty_example_outputs(): + text = PROMPTY.read_text(encoding="utf-8") + decoder = json.JSONDecoder() + return [ + decoder.raw_decode(text, text.index("{", match.end()))[0] for match in re.finditer(r"EXPECTED OUTPUT", text) + ] + + +@pytest.mark.unittest +class TestToolCallSuccessPrompty: + def test_every_example_output_is_valid_json_with_a_top_level_score(self): + outputs = prompty_example_outputs() + + assert len(outputs) == 14 + for output in outputs: + assert set(output) == {"reason", "properties", "score", "status"} + assert output["status"] == "completed" and output["score"] in (0, 1) + assert set(output["properties"]) == {"failed_tools"} + + +@pytest.mark.unittest +class TestToolCallSuccessScore: + @pytest.mark.parametrize( + "llm_output, expected", + [ + ({"score": 1, "status": "completed", "properties": {"failed_tools": ""}}, 1.0), + ({"score": 0, "status": "completed", "properties": {"failed_tools": "fetch_weather"}}, 0.0), + # Judge models such as gpt-4.1 and gpt-4o nested the score and status in properties. + ({"properties": {"failed_tools": "", "score": 1, "status": "completed"}}, 1.0), + ({"properties": {"failed_tools": "fetch_weather", "score": 0, "status": "completed"}}, 0.0), + # A top-level value wins over a nested one. + ({"score": 0, "properties": {"score": 1}}, 0.0), + ({"score": 1, "properties": {"score": 0}}, 1.0), + ({"score": None, "properties": {"score": 1}}, 1.0), + # Values that were already read as 0 or 1 still are. + ({"score": 1.0}, 1.0), + ({"score": "1"}, 1.0), + ({"score": "0"}, 0.0), + ({"score": True}, 1.0), + ({"score": False}, 0.0), + ], + ) + def test_the_score_is_read_from_the_top_level_or_from_properties(self, llm_output, expected): + result = evaluate_with({"reason": "Recorded reason.", **llm_output}) + + assert result["tool_call_success"] == expected + assert result["tool_call_success_score"] == expected + assert result["tool_call_success_result"] == ("pass" if expected == 1.0 else "fail") + assert result["tool_call_success_passed"] is (expected == 1.0) + assert result["tool_call_success_status"] == "completed" + assert result["tool_call_success_reason"] == "Recorded reason." + assert result["tool_call_success_properties"]["prompt_tokens"] == 11 + + def test_nested_keys_stay_in_the_returned_properties(self): + result = evaluate_with({"reason": "r", "properties": {"failed_tools": "", "score": 1, "status": "completed"}}) + + assert {"failed_tools": "", "score": 1, "status": "completed"}.items() <= ( + result["tool_call_success_properties"].items() + ) + + @pytest.mark.parametrize( + "llm_output", + [ + {"status": "skipped", "reason": "No tool calls."}, + {"properties": {"status": "skipped"}, "reason": "No tool calls."}, + {"properties": {"status": "skipped", "reason": "No tool calls."}}, + ], + ) + def test_a_skipped_status_is_not_applicable(self, llm_output): + result = evaluate_with(llm_output) + + assert result["tool_call_success_result"] == "not_applicable" + assert result["tool_call_success_reason"] == "Not applicable: No tool calls." + assert result["tool_call_success_status"] == "skipped" + assert result["tool_call_success"] is None + + def test_a_top_level_status_wins_over_a_nested_skip(self): + result = evaluate_with({"status": "completed", "score": 1, "properties": {"status": "skipped"}}) + + assert result["tool_call_success_status"] == "completed" + assert result["tool_call_success_result"] == "pass" + + @pytest.mark.parametrize( + "llm_output, shown", + [ + ({"reason": "No score anywhere.", "status": "completed"}, "None"), + ({"score": None, "properties": None}, "None"), + ({"score": None, "properties": {"failed_tools": ""}}, "None"), + ({"score": 0.5}, "0.5"), + ({"score": 2}, "2"), + ({"score": -1}, "-1"), + ({"score": math.nan}, "nan"), + ({"score": math.inf}, "inf"), + ({"score": "pass"}, "'pass'"), + ({"score": [1]}, "[1]"), + ({"score": {"value": 1}}, "{'value': 1}"), + ({"properties": {"score": 0.5}}, "0.5"), + ({"score": 0.5, "properties": {"score": 1}}, "0.5"), + ({"properties": "score: 1"}, "None"), + ], + ) + def test_an_output_without_a_score_of_0_or_1_is_an_error_not_a_failed_tool_call(self, llm_output, shown): + with pytest.raises(EvaluationException) as raised: + evaluate_with(llm_output) + + assert str(raised.value) == f"(InternalError) Invalid score value: {shown}. Expected 0 or 1." + assert raised.value.internal_message == "Invalid score value." + assert raised.value.blame == ErrorBlame.SYSTEM_ERROR + assert raised.value.category == ErrorCategory.FAILED_EXECUTION + assert raised.value.target == ErrorTarget.TOOL_CALL_SUCCESS_EVALUATOR + + def test_an_invalid_score_is_shown_shortened(self): + with pytest.raises(EvaluationException) as raised: + evaluate_with({"score": "x" * 500}) + + assert len(str(raised.value)) < 100 + + def test_evaluate_reports_a_row_without_a_score_as_an_error(self, tmp_path): + rows = [ + {"response": TOOL_CALL_RESPONSE, "case": "valid"}, + {"response": TOOL_CALL_RESPONSE, "case": "missing"}, + {"response": TOOL_CALL_RESPONSE, "case": "nested"}, + ] + data = tmp_path / "rows.jsonl" + data.write_text("\n".join(json.dumps(row) for row in rows), encoding="utf-8") + outputs = iter( + [ + {"score": 1, "status": "completed", "reason": "valid"}, + {"status": "completed", "reason": "missing"}, + {"properties": {"score": 1, "status": "completed"}, "reason": "nested"}, + ] + ) + evaluator = _ToolCallSuccessEvaluator(model_config=dict(CHAT_CONFIG)) + + async def flow(timeout, **kwargs): + return {"llm_output": next(outputs), **JUDGE_USAGE} + + evaluator._flow = flow + result = evaluate( + data=str(data), + evaluators={"tool_call_success": evaluator}, + evaluator_config={"tool_call_success": {"column_mapping": {"response": "${data.response}"}}}, + _use_run_submitter_client=True, + ) + reason, score = ( + "outputs.tool_call_success.tool_call_success_reason", + "outputs.tool_call_success.tool_call_success", + ) + completed = {row[reason]: row[score] for row in result["rows"] if isinstance(row.get(reason), str)} + errored = [row for row in result["rows"] if not isinstance(row.get(reason), str)] + + assert completed == {"valid": 1.0, "nested": 1.0} + assert len(errored) == 1 and pd.isna(errored[0][score]) + + +@pytest.mark.unittest +class TestToolCallSuccessWithoutToolCalls: + @pytest.mark.parametrize("config", [CHAT_CONFIG, DECISION_CONFIG], ids=["chat", "decision"]) + @pytest.mark.parametrize( + "response", + [ + TEXT_RESPONSE, + [{"role": "assistant", "content": "I can't check the weather."}], + [ + {"role": "system", "content": "Answer briefly."}, + {"role": "developer", "content": [{"type": "input_text", "text": "Be polite."}]}, + {"role": "assistant", "content": [{"type": "text", "text": "Hello."}]}, + ], + [ + { + "role": "assistant", + "content": [{"type": "mcp_approval_request", "name": "search", "arguments": "{}"}], + }, + { + "role": "tool", + "tool_call_id": "approval_1", + "content": [{"type": "mcp_approval_response", "approve": False}], + }, + ], + ], + ids=["text-items", "string-content", "every-text-role", "mcp-approval-only"], + ) + def test_a_response_without_tool_calls_is_not_applicable_without_a_judge_call(self, config, response): + evaluator = _ToolCallSuccessEvaluator(model_config=dict(config)) + if evaluator._decision is None: + evaluator._flow = AsyncMock(side_effect=AssertionError("The judge was called.")) + else: + evaluator._decision.evaluate = AsyncMock(side_effect=AssertionError("The judge was called.")) + + result = evaluator(response=response) + + assert result == { + "tool_call_success": None, + "tool_call_success_score": None, + "tool_call_success_passed": None, + "tool_call_success_result": "not_applicable", + "tool_call_success_reason": "Not applicable: No tool calls found in response.", + "tool_call_success_status": "skipped", + "tool_call_success_threshold": 1, + "tool_call_success_properties": None, + } + + @pytest.mark.parametrize( + "response", + [ + TOOL_CALL_RESPONSE, + [ + { + "role": "assistant", + "content": [ + { + "type": "function_call", + "tool_call_id": "call_1", + "name": "fetch_weather", + "arguments": {"city": "Paris"}, + }, + ], + }, + {"role": "assistant", "content": [{"type": "text", "text": "Done."}]}, + ], + "[TOOL_CALL] fetch_weather(city=Paris)", + "It is sunny in Paris.", + [ + { + "role": "assistant", + "content": '[TOOL_CALL] fetch_weather(city="Paris")\n[TOOL_RESULT] {"error": "timed out"}', + } + ], + [{"role": "assistant", "content": [{"type": "text", "text": '[TOOL_RESULT] {"error": "timed out"}'}]}], + [ + { + "role": "assistant", + "content": [ + {"type": "mcp_approval_request", "name": "search", "arguments": "{}"}, + { + "type": "tool_call", + "tool_call_id": "call_1", + "name": "fetch_weather", + "arguments": {"city": "Paris"}, + }, + ], + }, + {"role": "assistant", "content": [{"type": "text", "text": "Done."}]}, + ], + ], + ids=[ + "tool-call", + "function-call", + "tool-call-text", + "plain-text", + "trace-in-message", + "trace-in-text-item", + "approval-with-tool-call", + ], + ) + def test_tool_calls_and_text_responses_still_go_to_the_judge(self, response): + evaluator = judge({"score": 1, "status": "completed", "reason": "r"}) + + result = evaluator(response=response) + + assert evaluator._flow.await_count == 1 + assert result["tool_call_success_result"] == "pass" + + def test_a_decision_judge_still_gets_tool_calls(self): + evaluator = _ToolCallSuccessEvaluator(model_config=dict(DECISION_CONFIG)) + evaluator._decision.evaluate = AsyncMock(return_value={"tool_call_success": 1.0}) + + assert evaluator(response=TOOL_CALL_RESPONSE)["tool_call_success"] == 1.0 + state = evaluator._decision.evaluate.await_args.args[0] + assert "[TOOL_CALL] fetch_weather" in state["tool_calls"] + + @pytest.mark.parametrize( + "messages", + [ + [], + "text", + [{"role": "tool", "tool_call_id": "call_1", "content": [{"type": "tool_result", "tool_result": "ok"}]}], + [{"role": "assistant", "content": [{"type": "tool_call", "name": "t", "arguments": {}}]}], + [{"role": "assistant", "content": "Calling a tool.", "tool_calls": [{"id": "call_1"}]}], + [{"role": "assistant", "content": "Calling a tool.", "function_call": {"name": "t"}}], + [{"role": "assistant", "content": "Result.", "tool_call_id": "call_1"}], + [{"role": "function", "content": "Result."}], + [{"content": "No role."}], + [{"role": "assistant", "content": None}], + [{"role": "assistant", "content": [{"type": "mcp_call", "name": "t"}]}], + [{"role": "assistant", "content": [{"type": "image_url", "image_url": "https://example.com"}]}], + [{"role": "assistant", "content": ["text"]}], + ["text"], + [{"role": "assistant", "content": "[TOOL_CALL] fetch_weather()"}], + [{"role": "assistant", "content": [{"type": "output_text", "text": "[TOOL_RESULT] ok"}]}], + [{"role": "assistant", "content": [{"type": "text", "text": None}]}], + ], + ) + def test_only_recognized_text_messages_count_as_having_no_tool_calls(self, messages): + assert _has_no_tool_calls(messages) is False + + def test_text_messages_have_no_tool_calls(self): + assert _has_no_tool_calls(TEXT_RESPONSE) is True + assert _has_no_tool_calls([{"role": "assistant", "content": []}]) is True + + def test_mcp_approvals_are_not_tool_calls_but_a_tool_call_beside_one_is(self): + approval = {"type": "mcp_approval_request", "name": "search", "arguments": "{}"} + response = {"type": "mcp_approval_response", "approve": True} + text = {"type": "text", "text": "Searching."} + tool_call = {"type": "tool_call", "tool_call_id": "call_1", "name": "search", "arguments": {}} + + assert _has_no_tool_calls( + [ + {"role": "assistant", "content": [approval, text]}, + {"role": "tool", "tool_call_id": "approval_1", "content": [response]}, + ] + ) + assert not _has_no_tool_calls([{"role": "assistant", "content": [approval, tool_call]}]) + assert not _has_no_tool_calls( + [{"role": "tool", "tool_call_id": "call_1", "content": [response, {"type": "tool_result"}]}] + ) + # Tool calls in message fields count even beside approval-only content. + assert not _has_no_tool_calls( + [ + {"role": "assistant", "content": [approval], "tool_calls": [{"id": "call_1", "type": "function"}]}, + {"role": "assistant", "content": [text]}, + ] + ) + assert not _has_no_tool_calls([{"role": "assistant", "content": [approval], "function_call": {"name": "t"}}]) From 5c068e8d79bbe55fcac0acea28112fed2c56e78c Mon Sep 17 00:00:00 2001 From: Ahmad Qardahji Date: Tue, 3 Nov 2026 09:22:57 -0800 Subject: [PATCH 02/14] Share the chat token scope with Decision requests and drop the redundant retry list Decision requests now request tokens for https://ai.azure.com/.default, the scope Azure OpenAI chat requests use, so a credential shared by chat and Decision evaluators needs one token instead of one per scope. Decision endpoints accept both audiences. The explicit retry status list is removed: azure-core retries those statuses by default and unions an explicit list with its defaults, so retry behavior is unchanged. _get_token documents its absolute monotonic deadline, and the token cache module records why get_bearer_token_provider is not used in its place. --- .../azure-ai-evaluation/CHANGELOG.md | 4 ++ .../_evaluators/_common/_model_runners.py | 32 ++++++++++------ .../_legacy/_common/_token_cache.py | 7 ++++ .../test_builtin_decision_evaluators.py | 2 +- .../unittests/test_decision_model_support.py | 37 ++++++++++++------- .../unittests/test_prompty_token_reuse.py | 13 ++++--- 6 files changed, 64 insertions(+), 31 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index 4d1624a44789..01c124755637 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -15,6 +15,10 @@ - `_ToolCallSuccessEvaluator` no longer scores successful tool calls 0 when a chat judge model, such as gpt-4.1 or gpt-4o, puts `score` and `status` inside `properties`. The example outputs in its prompt are now valid JSON, and `score`, `status` and `reason` are read from `properties` when the output has none at the top level. A missing score is no longer reported as 0, and any score other than 0 or 1 raises a system-classified `EvaluationException`, as `_ToolInputAccuracyEvaluator` does. - `_ToolCallSuccessEvaluator` reports a list of messages that holds no tool calls, such as one that contains only text or only MCP approval messages, as not applicable without calling the chat or Decision judge model. Strings, text that contains `[TOOL_CALL]` or `[TOOL_RESULT]`, and messages with any other content are still sent to the judge model. +### Other Changes + +- Decision requests request tokens for the `https://ai.azure.com/.default` scope, the scope that Azure OpenAI chat model requests use, instead of `https://cognitiveservices.azure.com/.default`. Decision endpoints accept both, and a credential shared by chat and Decision evaluators now needs one token instead of one per scope. + ## 1.18.10 (2026-10-22) ### Features Added diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py index 02acc4920972..dac4c4feba99 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py @@ -24,6 +24,7 @@ from azure.core.pipeline.policies import AsyncRedirectPolicy, AsyncRetryPolicy, CustomHookPolicy from azure.core.pipeline.transport import AioHttpTransport +from azure.ai.evaluation._constants import TokenScope from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException from azure.ai.evaluation._legacy._common._token_cache import TOKENS as _TOKENS from ..._common.utils import construct_prompty_model_config, validate_model_config @@ -46,8 +47,9 @@ def value(self) -> str: DECISION_MODEL_TYPE = "decision" _DECISION_PATH = "/providers/microsoft/v1/systemone" -_DECISION_TOKEN_SCOPE = "https://cognitiveservices.azure.com/.default" -_DECISION_RETRY_STATUS_CODES = [408, 429, 500, 502, 503, 504, 529] +# The scope chat model requests use. Decision endpoints accept it, so a credential shared by chat and Decision +# evaluators needs one token. +_MODEL_TOKEN_SCOPE = TokenScope.COGNITIVE_SERVICES_MANAGEMENT.value LOGGER = logging.getLogger(__name__) _PREFIX = "azure.ai.evaluation.decision." _DECISION_CONFIG_KEYS = { @@ -671,23 +673,30 @@ async def _await_request(self, content: bytes, deadline: float, execution: Decis await asyncio.gather(task, return_exceptions=True) async def _get_token(self, deadline: float) -> Optional[str]: - """Return a bearer token, or None when the deadline passed while waiting for another request's token.""" + """Return a bearer token, or None when the deadline passed while waiting for another request's token. + + :param float deadline: Absolute ``time.monotonic()`` instant by which the request must complete, not a + duration or a wall-clock time; the caller derives it from the request timeout. + :return: The bearer token, or None when the deadline passed while this request waited for a token that + another request was acquiring. + :rtype: Optional[str] + """ credential = self._credential - token = _TOKENS.cached(credential, _DECISION_TOKEN_SCOPE) + token = _TOKENS.cached(credential, _MODEL_TOKEN_SCOPE) if token is None: # An acquisition that begins before a service rejects a token may return that token; the generation it # began under lets the cache refuse it. - generation = _TOKENS.generation(credential, _DECISION_TOKEN_SCOPE) + generation = _TOKENS.generation(credential, _MODEL_TOKEN_SCOPE) if inspect.iscoroutinefunction(credential.get_token): - token = await credential.get_token(_DECISION_TOKEN_SCOPE) - _TOKENS.store(credential, _DECISION_TOKEN_SCOPE, token, generation) + token = await credential.get_token(_MODEL_TOKEN_SCOPE) + _TOKENS.store(credential, _MODEL_TOKEN_SCOPE, token, generation) else: abandoned = threading.Event() try: token = await asyncio.to_thread( _TOKENS.fetch, credential, - _DECISION_TOKEN_SCOPE, + _MODEL_TOKEN_SCOPE, abandoned=abandoned, deadline=deadline, describe_failure=_token_failure, @@ -698,7 +707,7 @@ async def _get_token(self, deadline: float) -> Optional[str]: return None if inspect.isawaitable(token): token = await token - _TOKENS.store(credential, _DECISION_TOKEN_SCOPE, token, generation) + _TOKENS.store(credential, _MODEL_TOKEN_SCOPE, token, generation) value = getattr(token, "token", None) if not isinstance(value, str): raise _failed("Could not acquire a token for the decision model.", ErrorBlame.USER_ERROR) @@ -723,13 +732,14 @@ async def _post(self, content: bytes, *, deadline: float, execution: DecisionExe policies: Dict[str, Any] = { "custom_hook_policy": execution, + # azure-core retries 408, 429, 500, 502, 503, 504 and 506 through 998 by default; POST is opted in + # per request below. "retry_policy": AsyncRetryPolicy( retry_total=self._max_retries, retry_connect=self._max_retries, retry_read=self._max_retries, retry_status=self._max_retries, retry_backoff_factor=0.8, - retry_on_status_codes=_DECISION_RETRY_STATUS_CODES, ), "redirect_policy": AsyncRedirectPolicy(permit_redirects=False), } @@ -773,7 +783,7 @@ async def _post(self, content: bytes, *, deadline: float, execution: DecisionExe if status != 200: if status == 401 and token is not None: # A rejected token is not reused. - _TOKENS.discard(self._credential, _DECISION_TOKEN_SCOPE, token) + _TOKENS.discard(self._credential, _MODEL_TOKEN_SCOPE, token) blame = ErrorBlame.USER_ERROR if status in (400, 401, 403, 404, 422) else ErrorBlame.SYSTEM_ERROR provider_error = payload.get("error") if isinstance(payload, dict) else None code = provider_error.get("code") if isinstance(provider_error, dict) else None diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_token_cache.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_token_cache.py index 297d50f73d56..eca3b3ec3172 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_token_cache.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_token_cache.py @@ -12,6 +12,13 @@ that rejection is not stored, so a rejected token never re-enters the cache through a slower request. Requests that waited on an acquisition whose token was not cached, because of such a rejection or because of its expiry, use that token instead of each requesting another one. + +azure-identity's ``get_bearer_token_provider`` is not used because it keeps one token per provider instance with no +coordination between concurrent requests, so concurrent first requests each call the credential; because its +wrapper never sees a service's 401 response, so a rejected token is served until it expires; and because its +asynchronous form binds its lock to one event loop while each synchronous evaluator call runs on a new loop. +Sharing one token across evaluators, waiting with a deadline and under cancellation, and dropping a rejected token +need the coordination here. """ import inspect diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_builtin_decision_evaluators.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_builtin_decision_evaluators.py index 5c1e82fec93e..c8910abf4c10 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_builtin_decision_evaluators.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_builtin_decision_evaluators.py @@ -804,7 +804,7 @@ def test_qa_passes_the_credential_and_decision_controls_to_its_members(service): assert runner._max_retries == 1 and runner._cancellation_requested is _never_cancelled # Every member's request carries the forwarded credential's token, however many token requests that took. assert service.authorizations == ["Bearer decision-token"] * len(QA_KEYS) - assert credential.scopes and set(credential.scopes) == {("https://cognitiveservices.azure.com/.default",)} + assert credential.scopes and set(credential.scopes) == {("https://ai.azure.com/.default",)} assert [result[f"{key}_status"] for key in QA_KEYS] == ["completed"] * len(QA_KEYS) assert result["f1_score"] == pytest.approx(5 / 6) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index 26e3cfe0c7d7..3f96b524ecc9 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -29,6 +29,7 @@ from azure.core.exceptions import ServiceRequestError from azure.ai.evaluation import CoherenceEvaluator, FluencyEvaluator +from azure.ai.evaluation._constants import TokenScope from azure.ai.evaluation._evaluators._common import EvaluatorBase, ModelEvaluatorBase, PromptyEvaluatorBase from azure.ai.evaluation._evaluators._common import _model_runners from azure.ai.evaluation._evaluators._common._decision_strategy import ( @@ -153,7 +154,7 @@ def watch_token_queue(credential, requests): Each request reads the cache's attempt count before it queues, so a token request released after this event completes while every request is queued, whatever the thread scheduling. """ - entry = _model_runners._TOKENS._entry(credential, _model_runners._DECISION_TOKEN_SCOPE, create=True) + entry = _model_runners._TOKENS._entry(credential, _model_runners._MODEL_TOKEN_SCOPE, create=True) lock, queued, all_queued = entry.lock, set(), threading.Event() class QueueWatchingLock: @@ -338,7 +339,7 @@ def test_credential_acquires_a_scoped_bearer_token(self, client, asynchronous): credential.get_token = AsyncMock(return_value=token) if asynchronous else Mock(return_value=token) config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} run(DecisionModelRunner(config, credential)) - credential.get_token.assert_called_once_with("https://cognitiveservices.azure.com/.default") + credential.get_token.assert_called_once_with("https://ai.azure.com/.default") headers = client.calls[0]["headers"] assert headers["Authorization"] == "Bearer bearer-value" and "api-key" not in headers @@ -356,7 +357,8 @@ def test_bounded_retries_without_redirects(self, client): assert isinstance(retry, AsyncRetryPolicy) and retry.total_retries == 3 assert isinstance(redirect, AsyncRedirectPolicy) and redirect.allow is False - @pytest.mark.parametrize("status", _model_runners._DECISION_RETRY_STATUS_CODES) + # Statuses that azure-core's retry policy retries by default and a Decision endpoint can return. + @pytest.mark.parametrize("status", [408, 429, 500, 502, 503, 504, 529]) def test_transient_post_response_is_retried_without_retry_after(self, monkeypatch, status): from azure.ai.evaluation._http_utils import AsyncHttpPipeline @@ -604,7 +606,7 @@ def test_fresh_token_is_reused_across_requests_and_runners(self, client, asynchr first, second = DecisionModelRunner(self.CONFIG, credential), DecisionModelRunner(self.CONFIG, credential) for runner in (first, second, first): run(runner) - credential.get_token.assert_called_once_with("https://cognitiveservices.azure.com/.default") + credential.get_token.assert_called_once_with("https://ai.azure.com/.default") assert self.authorizations(client) == ["Bearer shared-token"] * 3 @pytest.mark.parametrize("lifetime", [None, 60], ids=["no-expiry", "near-expiry"]) @@ -669,7 +671,16 @@ def get_token(*_, **__): request.result(2 * WAIT_SECONDS) assert len(requested) == 1 assert self.authorizations(client) == ["Bearer uncached"] * 6 - assert _model_runners._TOKENS.cached(credential, _model_runners._DECISION_TOKEN_SCOPE) is None + assert _model_runners._TOKENS.cached(credential, _model_runners._MODEL_TOKEN_SCOPE) is None + + def test_requests_the_token_scope_chat_models_use(self, client): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + credential = Mock(get_token=Mock(return_value=self.access_token("shared"))) + run(DecisionModelRunner(self.CONFIG, credential)) + credential.get_token.assert_called_once_with("https://ai.azure.com/.default") + assert _model_runners._MODEL_TOKEN_SCOPE == TokenScope.COGNITIVE_SERVICES_MANAGEMENT + # The chat path keys the cache with the enum member; the entry is the same one. + assert _model_runners._TOKENS.cached(credential, TokenScope.COGNITIVE_SERVICES_MANAGEMENT).token == "shared" def test_rejected_token_is_not_reused(self, client): client.response = FakeResponse(status_code=401, body=b"{}") @@ -702,7 +713,7 @@ def test_a_token_the_credential_reissues_after_a_rejection_is_cached_again(self, runner = DecisionModelRunner(self.CONFIG, credential) with pytest.raises(EvaluationException, match="HTTP 401"): run(runner) - assert _model_runners._TOKENS.cached(credential, _model_runners._DECISION_TOKEN_SCOPE) is None + assert _model_runners._TOKENS.cached(credential, _model_runners._MODEL_TOKEN_SCOPE) is None # The service accepts the token now, for example after a role assignment propagated. The credential # re-issues the same token: it is requested once more, then reused again instead of once per request. client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) @@ -710,10 +721,10 @@ def test_a_token_the_credential_reissues_after_a_rejection_is_cached_again(self, run(runner) assert credential.get_token.call_count == 2 assert self.authorizations(client) == ["Bearer same"] * 4 - assert _model_runners._TOKENS.cached(credential, _model_runners._DECISION_TOKEN_SCOPE).token == "same" + assert _model_runners._TOKENS.cached(credential, _model_runners._MODEL_TOKEN_SCOPE).token == "same" def test_an_acquisition_that_began_before_two_rejections_is_not_stored(self): - scope = _model_runners._DECISION_TOKEN_SCOPE + scope = _model_runners._MODEL_TOKEN_SCOPE async def scenario(): entered, gate = asyncio.Event(), asyncio.Event() @@ -893,7 +904,7 @@ def get_token(*_, **__): credential = Mock(get_token=Mock(side_effect=get_token)) all_queued = watch_token_queue(credential, 4) - scope = _model_runners._DECISION_TOKEN_SCOPE + scope = _model_runners._MODEL_TOKEN_SCOPE with ThreadPoolExecutor(max_workers=4) as pool: try: requests = [ @@ -919,7 +930,7 @@ def test_a_token_rejected_before_its_request_resumes_is_not_stored_again(self, c client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) credential = Mock(get_token=Mock(return_value=self.access_token("shared"))) fetch = _model_runners._TOKENS.fetch - scope = _model_runners._DECISION_TOKEN_SCOPE + scope = _model_runners._MODEL_TOKEN_SCOPE def fetch_then_reject(*args, **kwargs): token = fetch(*args, **kwargs) @@ -1001,7 +1012,7 @@ def get_token(self, *_, **__): credential = Credential() run(DecisionModelRunner(self.CONFIG, credential)) - assert _model_runners._TOKENS.cached(credential, _model_runners._DECISION_TOKEN_SCOPE) is token + assert _model_runners._TOKENS.cached(credential, _model_runners._MODEL_TOKEN_SCOPE) is token reference = weakref.ref(credential) del credential gc.collect() @@ -1037,7 +1048,7 @@ def test_a_credential_that_returns_no_usable_token_is_a_user_error(self, client, def test_a_token_wait_ends_at_the_deadline_without_requesting_a_token(self): credential = Mock(get_token=Mock(return_value=self.access_token("unused"))) - scope = _model_runners._DECISION_TOKEN_SCOPE + scope = _model_runners._MODEL_TOKEN_SCOPE entry = _model_runners._TOKENS._entry(credential, scope, create=True) runner = DecisionModelRunner(self.CONFIG, credential) # Another request holds the credential's token request throughout. @@ -1052,7 +1063,7 @@ def test_a_token_wait_ends_at_the_deadline_without_requesting_a_token(self): def test_a_credential_without_weak_references_is_not_asked_after_the_deadline(self): requested = [] credential = SimpleNamespace(get_token=lambda *scopes, **_: requested.append(scopes)) - scope = _model_runners._DECISION_TOKEN_SCOPE + scope = _model_runners._MODEL_TOKEN_SCOPE assert _model_runners._TOKENS.fetch(credential, scope, deadline=time.monotonic() - 1) is None assert requested == [] diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_token_reuse.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_token_reuse.py index d67f5f225670..dc2bac8869e9 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_token_reuse.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_token_reuse.py @@ -19,6 +19,7 @@ from openai import AuthenticationError, InternalServerError import azure.ai.evaluation._evaluators._coherence as coherence +from azure.ai.evaluation._evaluators._common import _model_runners from azure.ai.evaluation._common.utils import construct_prompty_model_config, validate_model_config from azure.ai.evaluation._constants import TokenScope from azure.ai.evaluation._exceptions import EvaluationException @@ -28,7 +29,7 @@ from azure.ai.evaluation._legacy.prompty._exceptions import WrappedOpenAIError CHAT_SCOPE = TokenScope.COGNITIVE_SERVICES_MANAGEMENT -DECISION_SCOPE = "https://cognitiveservices.azure.com/.default" +DECISION_SCOPE = _model_runners._MODEL_TOKEN_SCOPE COHERENCE_PROMPTY = Path(coherence.__file__).parent / "coherence.prompty" CHAT_MODEL = construct_prompty_model_config( validate_model_config({"azure_endpoint": "https://example.openai.azure.com", "azure_deployment": "judge"}), @@ -200,13 +201,13 @@ async def scenario(): join_all([leader]) assert len(credential.scopes) == 1 - def test_chat_and_decision_tokens_are_kept_apart(self): - credential = Credential(access_token("chat")) + def test_chat_requests_reuse_a_token_a_decision_request_acquired(self): + credential = Credential(access_token("unused")) TOKENS.store(credential, DECISION_SCOPE, access_token("decision")) - assert request_token(credential) == "chat" - assert credential.scopes == [(CHAT_SCOPE,)] - assert TOKENS.cached(credential, DECISION_SCOPE).token == "decision" + assert request_token(credential) == "decision" + assert credential.scopes == [] + assert TOKENS.cached(credential, CHAT_SCOPE).token == "decision" def test_a_failed_token_request_raises_the_credential_error_and_is_retried(self): credential = Credential(ClientAuthenticationError("Please run az login."), access_token("recovered")) From 3881048ebb599a80c884b415a5a9c8e38a555d58 Mon Sep 17 00:00:00 2001 From: Ahmad Qardahji Date: Tue, 3 Nov 2026 10:08:39 -0800 Subject: [PATCH 03/14] Name the reason a failed Decision request was refused, and share only authentication failures with waiting requests A failed Decision request names a recognized reason in decision_model.error_kind of the error's properties, which is kept on failed result rows, and in the error message, for example "The decision model returned HTTP 422 (context_window_exceeded).". The provider states these reasons for an HTTP 422 only in text, which is never exposed: context_window_exceeded, request_token_limit_exceeded, question_limit_exceeded and invalid_question, recognized by the sentence's fixed opening so a quoted question identifier cannot change the kind. A request over max_request_bytes or max_state_question_bytes, which is not sent, has the kind request_bytes_exceeded. Requests that waited on a token request now share its failure only when it failed authentication, as on the chat path; after any other failure each waiting request makes its own attempt, so a transient failure fails only the request that hit it. --- .../azure-ai-evaluation/CHANGELOG.md | 3 +- .../_batch_run/_run_submitter_client.py | 3 +- .../_evaluators/_common/_model_runners.py | 76 +++++-- .../ai/evaluation/_legacy/prompty/_prompty.py | 2 + .../test_decision_batch_persistence.py | 81 ++++++++ .../unittests/test_decision_model_support.py | 187 ++++++++++++++++-- 6 files changed, 319 insertions(+), 33 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index 01c124755637..e42e1bf44079 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -8,10 +8,11 @@ - `QAEvaluator` accepts a Decision model configuration and passes `credential` and the Decision execution controls to its model-based evaluators, which run on separate threads, so the credential must be synchronous. With a chat model they are not passed, as before. - The binary evaluators (task adherence, task completion, tool call success, tool input accuracy, tool output utilization and tool selection) accept a `probability_threshold` keyword. For Decision models it sets the probability at or above which the score is 1 (default 0.75) and overrides the value in `model_config`; chat models ignore it with a warning. - Conversation inputs are handled as before with a Decision model: evaluators that accept them score each user and assistant turn separately and aggregate the results. The catalog's whole-conversation rubrics are not included. +- A failed Decision request names a recognized reason in `decision_model.error_kind` of the error's properties, which is kept on failed result rows, and in the error message, for example `The decision model returned HTTP 422 (context_window_exceeded).` The provider states these reasons for an HTTP 422 only in text, which is never exposed: `context_window_exceeded` when the evidence and one question exceed the model's context window, `request_token_limit_exceeded` when the evidence and all questions exceed the model's request limit, `question_limit_exceeded` and `invalid_question`. A request over `max_request_bytes` or `max_state_question_bytes`, which is not sent, has the kind `request_bytes_exceeded`. ### Bugs Fixed -- Decision requests, and Azure OpenAI chat model requests that authenticate with Microsoft Entra ID, reuse a credential's token until five minutes before it expires, as azure-core's bearer token policy does, instead of requesting a token for every request. Concurrent evaluations that share a synchronous credential wait for one token request, without blocking their event loops. With developer credentials such as `AzureCliCredential`, which start a process per request, concurrent evaluations were slow and could fail. Decision requests that waited on a failed token request share its failure instead of each requesting a token again, and chat requests share an authentication failure the same way. A token rejected with HTTP 401 is dropped and requested again; a token whose request began before the rejection is not kept for reuse, and requests that waited on it use it rather than each requesting another one. The default credential that a chat evaluator creates when no `credential` is passed is shared the same way. Tokens are kept per scope, and the cache never keeps a credential alive. A token without an expiry, or that expires within five minutes, is not kept, but the requests that waited for it use it. +- Decision requests, and Azure OpenAI chat model requests that authenticate with Microsoft Entra ID, reuse a credential's token until five minutes before it expires, as azure-core's bearer token policy does, instead of requesting a token for every request. Concurrent evaluations that share a synchronous credential wait for one token request, without blocking their event loops. With developer credentials such as `AzureCliCredential`, which start a process per request, concurrent evaluations were slow and could fail. Requests that waited on a token request that failed authentication share its failure instead of each requesting a token again; after any other failure, such as a token service outage, each waiting request makes its own attempt, so a transient failure fails only the request that hit it. Decision and chat requests behave the same way. A token rejected with HTTP 401 is dropped and requested again; a token whose request began before the rejection is not kept for reuse, and requests that waited on it use it rather than each requesting another one. The default credential that a chat evaluator creates when no `credential` is passed is shared the same way. Tokens are kept per scope, and the cache never keeps a credential alive. A token without an expiry, or that expires within five minutes, is not kept, but the requests that waited for it use it. - `_ToolCallSuccessEvaluator` no longer scores successful tool calls 0 when a chat judge model, such as gpt-4.1 or gpt-4o, puts `score` and `status` inside `properties`. The example outputs in its prompt are now valid JSON, and `score`, `status` and `reason` are read from `properties` when the output has none at the top level. A missing score is no longer reported as 0, and any score other than 0 or 1 raises a system-classified `EvaluationException`, as `_ToolInputAccuracyEvaluator` does. - `_ToolCallSuccessEvaluator` reports a list of messages that holds no tool calls, such as one that contains only text or only MCP approval messages, as not applicable without calling the chat or Decision judge model. Strings, text that contains `[TOOL_CALL]` or `[TOOL_RESULT]`, and messages with any other content are still sent to the judge model. diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py index debf679e61db..63f1d9960485 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py @@ -22,7 +22,7 @@ from ..._evaluate._utils import _has_aggregator from ..._constants import Prefixes, PF_BATCH_TIMEOUT_SEC from ..._exceptions import EvaluationException -from ..._evaluators._common._model_runners import _is_finite_number, validated_execution +from ..._evaluators._common._model_runners import DECISION_ERROR_KINDS, _is_finite_number, validated_execution from .._utils import get_int_env_var as get_int @@ -56,6 +56,7 @@ def _decision_error_metadata(exception: EvaluationException) -> Dict[str, Any]: ("backend", ("foundry_decision", "system_one")), ("protocol", ("shared_sdk", "system_one")), ("result_usage_scope", ("reported_attempts_only", "final_response_only", "reported_turns_only")), + ("error_kind", DECISION_ERROR_KINDS), ): value = native.get(key) if isinstance(value, str) and value in accepted: diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py index dac4c4feba99..04acccdbbb29 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py @@ -21,6 +21,7 @@ from typing import Any, Callable, Dict, List, Optional from urllib.parse import urlsplit +from azure.core.exceptions import ClientAuthenticationError from azure.core.pipeline.policies import AsyncRedirectPolicy, AsyncRetryPolicy, CustomHookPolicy from azure.core.pipeline.transport import AioHttpTransport @@ -63,6 +64,23 @@ def value(self) -> str: _DEFAULT_PROBABILITY_THRESHOLD = 0.75 # The service adds api_version to every resolved model configuration; the decision endpoint does not use it. _IGNORED_CONFIG_KEYS = {"api_version"} +_ERROR_CODE = re.compile(r"[A-Za-z][A-Za-z0-9_.-]{0,127}") +# A 422 states its reason only in a ``detail`` sentence, which is never surfaced. These reasons are recognized by the +# provider's fixed sentence openings, so a quoted question identifier later in the sentence cannot change the kind, and +# reported as a bounded ``decision_model.error_kind``; any other sentence is ignored. Only the sentence's first +# characters are read, which hold every opening, so a long question identifier cannot hide the reason. +_PROVIDER_DETAIL_KINDS = ( + (re.compile(r"invalid model request: Question prompt exceeds the \d+-token context limit\b"), + "context_window_exceeded"), + (re.compile(r"invalid TypeSafe request: request is about \d+ tokens; the limit for the state plus all questions is \d+"), + "request_token_limit_exceeded"), + (re.compile(r"invalid TypeSafe request: request has \d+ questions; the limit is \d+"), "question_limit_exceeded"), + (re.compile(r"invalid TypeSafe request: invalid [a-z_]+ for question \""), "invalid_question"), +) +_MAX_PROVIDER_DETAIL_CHARS = 1024 +# Reported when a request exceeds the caller's max_request_bytes or max_state_question_bytes and is not sent. +_REQUEST_BYTES_EXCEEDED = "request_bytes_exceeded" +DECISION_ERROR_KINDS = tuple(kind for _, kind in _PROVIDER_DETAIL_KINDS) + (_REQUEST_BYTES_EXCEEDED,) def is_decision_model_config(model_config: Any) -> bool: @@ -344,16 +362,15 @@ def _is_finite_number(value: Any) -> bool: return False -def _token_failure(error: BaseException) -> Dict[str, Any]: - """Describe a failed token request without keeping the error, whose traceback references the credential.""" - if isinstance(error, EvaluationException): - return { - "message": error.message, - "internal_message": error.internal_message, - "target": error.target, - "category": error.category, - "blame": error.blame, - } +def _token_failure(error: BaseException) -> Optional[Dict[str, Any]]: + """Describe an authentication failure for the requests that waited on it, or return None for any other failure. + + As on the chat path, only an authentication failure is shared: any other failure, such as a token service outage, + is not, so each waiting request makes its own attempt and one transient failure fails only the request that hit + it. The description never keeps the error, whose traceback references the credential. + """ + if not isinstance(error, ClientAuthenticationError): + return None message = "Could not acquire a token for the decision model." return { "message": message, @@ -364,6 +381,26 @@ def _token_failure(error: BaseException) -> Dict[str, Any]: } +def _provider_error_code(payload: Any) -> Optional[str]: + """Return the provider's nested ``error.code`` when it is a bounded identifier, otherwise None.""" + provider_error = payload.get("error") if isinstance(payload, dict) else None + code = provider_error.get("code") if isinstance(provider_error, dict) else None + return code if isinstance(code, str) and _ERROR_CODE.fullmatch(code) is not None else None + + +def _provider_error_kind(status: int, payload: Any) -> Optional[str]: + """Name a 422 that states its reason only in a ``detail`` sentence, from the sentence's fixed opening. + + For example ``{"detail": "invalid model request: Question prompt exceeds the 32768-token context limit"}`` is + ``context_window_exceeded``. Any other status or unrecognized sentence returns None and is never surfaced. + """ + detail = payload.get("detail") if status == 422 and isinstance(payload, dict) else None + if not isinstance(detail, str): + return None + opening = detail[:_MAX_PROVIDER_DETAIL_CHARS] + return next((kind for phrase, kind in _PROVIDER_DETAIL_KINDS if phrase.match(opening)), None) + + def _probability(value: Any) -> float: if not _is_finite_number(value): raise ValueError("Decision probabilities must be finite numbers.") @@ -439,6 +476,10 @@ class DecisionModelRunner: it stops awaiting synchronous credentials but cannot stop their worker threads. Byte limits measure compact unescaped UTF-8 JSON, with ``max_state_question_bytes`` applying to the largest individual pair. ``max_response_bytes`` limits acceptance before parsing, not transport allocation. + A failed request names a recognized reason in ``decision_model.error_kind`` of the error's properties: + ``context_window_exceeded`` when the evidence and one question exceed the model's context window, + ``request_token_limit_exceeded``, ``question_limit_exceeded`` or ``invalid_question`` for the provider's other + rejections, and ``request_bytes_exceeded`` for a request over the byte limits, which is not sent. """ def __init__( @@ -607,6 +648,7 @@ async def __call__(self, *, state: Any, questions: Dict[str, Dict[str, Any]], ti if (self._max_request_bytes is not None and len(content) > self._max_request_bytes) or ( self._max_state_question_bytes is not None and state_question_bytes > self._max_state_question_bytes ): + properties["decision_model"]["error_kind"] = _REQUEST_BYTES_EXCEEDED raise _failed("The decision request exceeds the configured byte budget.", ErrorBlame.USER_ERROR) try: payload = await asyncio.wait_for( @@ -693,6 +735,8 @@ async def _get_token(self, deadline: float) -> Optional[str]: else: abandoned = threading.Event() try: + # A request waiting for another request's token holds a default-executor thread while it waits; + # past the executor's size, further requests queue for a thread before they reach the cache. token = await asyncio.to_thread( _TOKENS.fetch, credential, @@ -785,15 +829,13 @@ async def _post(self, content: bytes, *, deadline: float, execution: DecisionExe # A rejected token is not reused. _TOKENS.discard(self._credential, _MODEL_TOKEN_SCOPE, token) blame = ErrorBlame.USER_ERROR if status in (400, 401, 403, 404, 422) else ErrorBlame.SYSTEM_ERROR - provider_error = payload.get("error") if isinstance(payload, dict) else None - code = provider_error.get("code") if isinstance(provider_error, dict) else None - diagnostic = ( - f" ({code})" - if isinstance(code, str) and re.fullmatch(r"[A-Za-z][A-Za-z0-9_.-]{0,127}", code) is not None - else "" - ) + code = _provider_error_code(payload) + kind = _provider_error_kind(status, payload) + diagnostic = f" ({code or kind})" if code or kind else "" error = _failed(f"The decision model returned HTTP {status}{diagnostic}.", blame) error.properties = self._response_properties(payload) + if kind is not None: + error.properties["decision_model"]["error_kind"] = kind raise error return payload diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py index e194b64c24b4..ad944656b7a6 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py @@ -485,6 +485,8 @@ async def __call__(self) -> str: else: abandoned = threading.Event() try: + # A request waiting for another request's token holds a default-executor thread while it waits; + # past the executor's size, further requests queue for a thread before they reach the cache. token = await asyncio.to_thread( TOKENS.fetch, credential, diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py index b5ecfc6c3e72..e7581365e537 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py @@ -222,6 +222,87 @@ def test_failed_metadata_rejects_untyped_and_unrecognized_provenance(self): ) assert _decision_error_metadata(error)["properties"] == {"decision_model": {}} + @pytest.mark.parametrize( + "kind,kept", + [ + ("context_window_exceeded", True), + ("request_token_limit_exceeded", True), + ("question_limit_exceeded", True), + ("invalid_question", True), + ("request_bytes_exceeded", True), + ("DeploymentNotFound", False), + ("context_window_exceeded\nprivate", False), + (42, False), + (None, False), + ], + ) + def test_failed_metadata_keeps_only_a_known_error_kind(self, kind, kept): + error = EvaluationException("synthetic failure", properties={"decision_model": {"error_kind": kind}}) + assert _decision_error_metadata(error)["properties"] == { + "decision_model": {"error_kind": kind} if kept else {} + } + + @pytest.mark.parametrize( + "detail,max_request_bytes,kind,message", + [ + ( + "invalid model request: Question prompt exceeds the 32768-token context limit including the decision " + "token", + None, + "context_window_exceeded", + "The decision model returned HTTP 422 (context_window_exceeded).", + ), + (None, 16, "request_bytes_exceeded", "The decision request exceeds the configured byte budget."), + ], + ids=["provider", "byte_limit"], + ) + def test_failure_reason_survives_batch_conversion( + self, batch_client, tmp_path, monkeypatch, detail, max_request_bytes, kind, message + ): + sent = [] + + class Response(HttpResponse): + def __init__(self, request): + super().__init__(request, None) + self.status_code = 422 + self.headers = {} + + def body(self): + return json.dumps({"detail": detail}).encode() + + class Transport(AsyncHttpTransport): + async def open(self): + pass + + async def close(self): + pass + + async def __aexit__(self, *_): + await self.close() + + async def send(self, request, **kwargs): + sent.append(request) + return Response(request) + + monkeypatch.setattr( + _model_runners, "get_async_http_client", lambda **_: AsyncHttpPipeline(transport=Transport()) + ) + + async def judge(query): + runner = DecisionModelRunner(CONFIG, max_retries=0, max_request_bytes=max_request_bytes) + return await strategy(runner).evaluate({"query": query}) + + _, rows = convert_batch(batch_client, judge, tmp_path) + result = next(item for item in rows[0]["results"] if item["name"] == "judge") + assert result["status"] == "error" and result["score"] is None + assert result["sample"]["error"]["blame"] == "UserError" + assert message in result["sample"]["error"]["message"] + native = result["properties"]["decision_model"] + assert native["error_kind"] == kind + assert native["execution"]["last_http_status"] == (422 if detail else None) + assert len(sent) == (1 if detail else 0) + assert "32768-token" not in json.dumps(rows) + @pytest.mark.parametrize("members", [5, 6]) @pytest.mark.parametrize("case", ["completed", "skipped", "mixed", "semantic_error"]) @pytest.mark.parametrize("primary_first", [False, True]) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index 3f96b524ecc9..fd8d977b6d12 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -26,7 +26,7 @@ from azure.core.credentials import AccessToken from azure.core.pipeline.policies import AsyncRedirectPolicy, AsyncRetryPolicy from azure.core.pipeline.transport import AsyncHttpTransport, HttpResponse -from azure.core.exceptions import ServiceRequestError +from azure.core.exceptions import ClientAuthenticationError, ServiceRequestError from azure.ai.evaluation import CoherenceEvaluator, FluencyEvaluator from azure.ai.evaluation._constants import TokenScope @@ -494,6 +494,102 @@ def test_provider_http_error_code_preserves_classification_and_accepted_metadata else {"decision_model": {"deployment": "Decision-1"}} ) + @pytest.mark.parametrize( + "detail,kind", + [ + ( + "invalid model request: Question prompt exceeds the 32768-token context limit including the decision " + "token", + "context_window_exceeded", + ), + ( + "invalid TypeSafe request: request is about 75108 tokens; the limit for the state plus all questions " + "is 64000", + "request_token_limit_exceeded", + ), + ("invalid TypeSafe request: request has 256 questions; the limit is 255", "question_limit_exceeded"), + ('invalid TypeSafe request: invalid instructions for question "private-key": value must not be empty', + "invalid_question"), + ('invalid TypeSafe request: invalid criteria for question "q": value must not be empty', "invalid_question"), + ("invalid TypeSafe request: private-message test-key-value", None), + ("invalid model request: Question prompt exceeds the 32768-token context limit " + "x" * 5000, + "context_window_exceeded"), + ('invalid TypeSafe request: invalid instructions for question "' + "q" * 2000 + '": value must not be empty', + "invalid_question"), + ("x" * 1100 + "invalid model request: Question prompt exceeds the 32768-token context limit", None), + ("", None), + ], + ) + def test_provider_rejection_reason_is_named_by_a_known_kind_without_its_text(self, client, detail, kind): + client.response = FakeResponse(422, json.dumps({"detail": detail}).encode()) + with pytest.raises(EvaluationException) as raised: + run(DecisionModelRunner(DECISION_CONFIG, max_retries=0)) + diagnostic = f" ({kind})" if kind else "" + assert str(raised.value) == f"(UserError) The decision model returned HTTP 422{diagnostic}." + assert raised.value.blame == ErrorBlame.USER_ERROR + native = raised.value.properties["decision_model"] + assert native.get("error_kind") == kind + persisted = json.dumps(raised.value.properties) + str(raised.value) + raised.value.internal_message + assert "private" not in persisted and "test-key-value" not in persisted and "32768" not in persisted + assert len(client.calls) == 1 + + @pytest.mark.parametrize( + "body", + [ + {"detail": ["exceeds the 1-token context limit"]}, + {"detail": {"message": "exceeds the 1-token context limit"}}, + {"error": "exceeds the 1-token context limit"}, + ["exceeds the 1-token context limit"], + ], + ) + def test_malformed_provider_rejection_detail_has_no_kind(self, client, body): + client.response = FakeResponse(422, json.dumps(body).encode()) + with pytest.raises(EvaluationException) as raised: + run(DecisionModelRunner(DECISION_CONFIG, max_retries=0)) + assert str(raised.value) == "(UserError) The decision model returned HTTP 422." + assert "error_kind" not in raised.value.properties["decision_model"] + + def test_provider_error_code_names_the_message_and_only_a_422_has_a_kind(self, client): + body = { + "error": {"code": "unsupported_request_argument", "message": "private-message"}, + "detail": "invalid model request: Question prompt exceeds the 32768-token context limit", + } + client.response = FakeResponse(400, json.dumps(body).encode()) + with pytest.raises(EvaluationException) as raised: + run(DecisionModelRunner(DECISION_CONFIG, max_retries=0)) + assert str(raised.value) == "(UserError) The decision model returned HTTP 400 (unsupported_request_argument)." + assert "error_kind" not in raised.value.properties["decision_model"] + assert "private-message" not in json.dumps(raised.value.properties) + + @pytest.mark.parametrize( + "detail", + [ + 'invalid TypeSafe request: invalid instructions for question "exceeds the 32768-token context limit": ' + "value must not be empty", + 'invalid TypeSafe request: invalid criteria for question "request has 300 questions; the limit is 255": ' + "value must not be empty", + ], + ) + def test_a_quoted_question_identifier_cannot_change_the_kind(self, client, detail): + client.response = FakeResponse(422, json.dumps({"detail": detail}).encode()) + with pytest.raises(EvaluationException) as raised: + run(DecisionModelRunner(DECISION_CONFIG, max_retries=0)) + assert raised.value.properties["decision_model"]["error_kind"] == "invalid_question" + + @pytest.mark.parametrize( + "limits", + [{"max_request_bytes": 64}, {"max_state_question_bytes": 64}], + ids=["request", "state_question"], + ) + def test_request_over_the_byte_limits_names_its_reason_and_is_not_sent(self, client, limits): + with pytest.raises(EvaluationException, match="exceeds the configured byte budget") as raised: + run(DecisionModelRunner(DECISION_CONFIG, **limits), state={"response": "x" * 200}) + assert raised.value.blame == ErrorBlame.USER_ERROR + native = raised.value.properties["decision_model"] + assert native["error_kind"] == "request_bytes_exceeded" + assert native["execution"]["attempt_count"] == 0 + assert not client.calls + @pytest.mark.parametrize( "payload", [ @@ -844,7 +940,33 @@ def request(name, timeout): assert "timed out" in outcomes["waiter"] assert "Could not acquire a token" in outcomes["leader"] - def test_requests_waiting_on_a_failed_token_request_share_its_failure(self, client): + @pytest.mark.parametrize( + "failure,token_requests,blame,message", + [ + pytest.param( + lambda: ClientAuthenticationError("Please run 'az login' (private-detail)"), + 1, + ErrorBlame.USER_ERROR, + "Could not acquire a token", + id="authentication-shared-at-once", + ), + pytest.param( + lambda: EvaluationException( + message="Service-owned Decision credential acquisition failed (RuntimeError, status 5xx).", + blame=ErrorBlame.SYSTEM_ERROR, + category=ErrorCategory.FAILED_EXECUTION, + target=ErrorTarget.EVALUATE, + ), + 4, + ErrorBlame.SYSTEM_ERROR, + "acquisition failed", + id="other-not-shared-each-request-tries", + ), + ], + ) + def test_requests_waiting_on_a_failed_token_request_share_its_failure( + self, client, failure, token_requests, blame, message + ): client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) entered, release = threading.Event(), threading.Event() requested = [] @@ -853,12 +975,7 @@ def get_token(*_, **__): requested.append(1) entered.set() release.wait() - raise EvaluationException( - message="Service-owned Decision credential acquisition failed (RuntimeError, status unknown).", - blame=ErrorBlame.SYSTEM_ERROR, - category=ErrorCategory.FAILED_EXECUTION, - target=ErrorTarget.EVALUATE, - ) + raise failure() credential = Mock(get_token=Mock(side_effect=get_token)) all_queued = watch_token_queue(credential, 4) @@ -882,14 +999,56 @@ def request(name): release.set() for thread in [leader, *waiters]: thread.join(WAIT_SECONDS) - # The queued requests fail with the same typed error instead of each starting another request. - assert len(requested) == 1 and len(errors) == 4 - assert all( - error.blame == ErrorBlame.SYSTEM_ERROR and "acquisition failed" in str(error) for error in errors.values() - ) + # Queued requests share an authentication failure instead of each starting another request; any other failure is + # not shared, so each request makes its own attempt and keeps its own typed error. + assert len(requested) == token_requests and len(errors) == 4 + assert all(error.blame == blame and message in str(error) for error in errors.values()) + assert all("private-detail" not in str(error) for error in errors.values()) credential.get_token.side_effect = lambda *_, **__: self.access_token("recovered") run(DecisionModelRunner(self.CONFIG, credential)) - assert len(requested) == 1 and self.authorizations(client) == ["Bearer recovered"] + assert credential.get_token.call_count == token_requests + 1 and self.authorizations(client) == ["Bearer recovered"] + + def test_requests_waiting_on_a_transient_token_failure_recover_with_one_more_request(self, client): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + entered, release = threading.Event(), threading.Event() + requested = [] + + def get_token(*_, **__): + requested.append(1) + if len(requested) == 1: + entered.set() + release.wait() + raise RuntimeError("token service unavailable") + return self.access_token("recovered") + + credential = Mock(get_token=Mock(side_effect=get_token)) + all_queued = watch_token_queue(credential, 4) + outcomes = {} + + def request(name): + try: + run(DecisionModelRunner(self.CONFIG, credential), timeout=2 * WAIT_SECONDS) + outcomes[name] = "completed" + except EvaluationException as error: + outcomes[name] = str(error) + + leader = threading.Thread(target=request, args=("leader",)) + waiters = [threading.Thread(target=request, args=(f"waiter-{index}",)) for index in range(3)] + try: + leader.start() + assert entered.wait(WAIT_SECONDS) + for waiter in waiters: + waiter.start() + assert all_queued.wait(WAIT_SECONDS) + finally: + release.set() + for thread in [leader, *waiters]: + thread.join(WAIT_SECONDS) + # Only the request whose token request failed fails; the next request recovers the token for the others. + assert len(requested) == 2 + assert "Could not acquire a token" in outcomes["leader"] + assert [outcomes[f"waiter-{index}"] for index in range(3)] == ["completed"] * 3 + assert self.authorizations(client) == ["Bearer recovered"] * 3 def test_requests_waiting_on_a_token_kept_out_of_the_cache_use_it(self, client): client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) From 8ac9c3a42e0e159309b554a9551b7d5914a57b5d Mon Sep 17 00:00:00 2001 From: Ahmad Qardahji Date: Tue, 3 Nov 2026 11:46:17 -0800 Subject: [PATCH 04/14] Keep Decision on the Cognitive Services scope and keep token failures within each kind of request Decision endpoints accept both audiences, but the hosted evaluation service selects the identity for Decision calls by token audience: for https://ai.azure.com it forwards the caller's own token, and for the Cognitive Services audience it exchanges that token or uses the project managed identity. With the chat scope, single-tenant projects would call Decision as the end user instead of the project managed identity, and the service's worker tests fail. Decision requests go back to https://cognitiveservices.azure.com/.default and their own cached token, and the redundant retry status list stays removed. The token cache records a failure per failure formatter and exception type, with the attempt that ended in it, so a request shares only a failure of its own kind that ended after it queued, and a successful token request clears them. A request could otherwise replay another kind's failure with its own exception type: a Decision description raised as a chat ClientAuthenticationError, or a chat authentication failure raised as an EvaluationException with an unknown blame. A credential that implements AsyncTokenCredential but whose get_token is a plain function returning an awaitable was asked on a worker thread, where a get_token that needs the running event loop failed. A credential whose get_token is a coroutine function, or that implements AsyncTokenCredential, is now asked on the caller's event loop, as released chat requests did. --- .../azure-ai-evaluation/CHANGELOG.md | 6 +- .../_evaluators/_common/_model_runners.py | 32 +-- .../_legacy/_common/_token_cache.py | 64 +++-- .../ai/evaluation/_legacy/prompty/_prompty.py | 21 +- .../test_builtin_decision_evaluators.py | 2 +- .../unittests/test_decision_model_support.py | 35 ++- .../unittests/test_prompty_token_reuse.py | 220 +++++++++++++++++- 7 files changed, 311 insertions(+), 69 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index e42e1bf44079..6bb455b3f68a 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -12,14 +12,10 @@ ### Bugs Fixed -- Decision requests, and Azure OpenAI chat model requests that authenticate with Microsoft Entra ID, reuse a credential's token until five minutes before it expires, as azure-core's bearer token policy does, instead of requesting a token for every request. Concurrent evaluations that share a synchronous credential wait for one token request, without blocking their event loops. With developer credentials such as `AzureCliCredential`, which start a process per request, concurrent evaluations were slow and could fail. Requests that waited on a token request that failed authentication share its failure instead of each requesting a token again; after any other failure, such as a token service outage, each waiting request makes its own attempt, so a transient failure fails only the request that hit it. Decision and chat requests behave the same way. A token rejected with HTTP 401 is dropped and requested again; a token whose request began before the rejection is not kept for reuse, and requests that waited on it use it rather than each requesting another one. The default credential that a chat evaluator creates when no `credential` is passed is shared the same way. Tokens are kept per scope, and the cache never keeps a credential alive. A token without an expiry, or that expires within five minutes, is not kept, but the requests that waited for it use it. +- Decision requests, and Azure OpenAI chat model requests that authenticate with Microsoft Entra ID, reuse a credential's token until five minutes before it expires, as azure-core's bearer token policy does, instead of requesting a token for every request. Concurrent evaluations that share a synchronous credential wait for one token request, without blocking their event loops. A credential whose `get_token` is a coroutine function, or that implements `AsyncTokenCredential`, is asked for its token on the caller's event loop, and any other credential on a worker thread. With developer credentials such as `AzureCliCredential`, which start a process per request, concurrent evaluations were slow and could fail. Requests that waited on a token request that failed authentication share its failure instead of each requesting a token again; after any other failure, such as a token service outage, each waiting request makes its own attempt, so a transient failure fails only the request that hit it. Decision and chat requests behave the same way. A token rejected with HTTP 401 is dropped and requested again; a token whose request began before the rejection is not kept for reuse, and requests that waited on it use it rather than each requesting another one. The default credential that a chat evaluator creates when no `credential` is passed is shared the same way. Tokens are kept per scope, and the cache never keeps a credential alive. A token without an expiry, or that expires within five minutes, is not kept, but the requests that waited for it use it. - `_ToolCallSuccessEvaluator` no longer scores successful tool calls 0 when a chat judge model, such as gpt-4.1 or gpt-4o, puts `score` and `status` inside `properties`. The example outputs in its prompt are now valid JSON, and `score`, `status` and `reason` are read from `properties` when the output has none at the top level. A missing score is no longer reported as 0, and any score other than 0 or 1 raises a system-classified `EvaluationException`, as `_ToolInputAccuracyEvaluator` does. - `_ToolCallSuccessEvaluator` reports a list of messages that holds no tool calls, such as one that contains only text or only MCP approval messages, as not applicable without calling the chat or Decision judge model. Strings, text that contains `[TOOL_CALL]` or `[TOOL_RESULT]`, and messages with any other content are still sent to the judge model. -### Other Changes - -- Decision requests request tokens for the `https://ai.azure.com/.default` scope, the scope that Azure OpenAI chat model requests use, instead of `https://cognitiveservices.azure.com/.default`. Decision endpoints accept both, and a credential shared by chat and Decision evaluators now needs one token instead of one per scope. - ## 1.18.10 (2026-10-22) ### Features Added diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py index 04acccdbbb29..8150c458e74a 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py @@ -25,9 +25,8 @@ from azure.core.pipeline.policies import AsyncRedirectPolicy, AsyncRetryPolicy, CustomHookPolicy from azure.core.pipeline.transport import AioHttpTransport -from azure.ai.evaluation._constants import TokenScope from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException -from azure.ai.evaluation._legacy._common._token_cache import TOKENS as _TOKENS +from azure.ai.evaluation._legacy._common._token_cache import TOKENS as _TOKENS, is_async_credential from ..._common.utils import construct_prompty_model_config, validate_model_config from ..._http_utils import get_async_http_client @@ -48,9 +47,10 @@ def value(self) -> str: DECISION_MODEL_TYPE = "decision" _DECISION_PATH = "/providers/microsoft/v1/systemone" -# The scope chat model requests use. Decision endpoints accept it, so a credential shared by chat and Decision -# evaluators needs one token. -_MODEL_TOKEN_SCOPE = TokenScope.COGNITIVE_SERVICES_MANAGEMENT.value +# Decision endpoints also accept the chat scope, https://ai.azure.com/.default, but the hosted evaluation service +# relies on this audience to call Decision with its exchanged or managed identity token rather than the caller's own +# token, so Decision requests keep this scope and their own cached token. +_DECISION_TOKEN_SCOPE = "https://cognitiveservices.azure.com/.default" LOGGER = logging.getLogger(__name__) _PREFIX = "azure.ai.evaluation.decision." _DECISION_CONFIG_KEYS = { @@ -717,6 +717,9 @@ async def _await_request(self, content: bytes, deadline: float, execution: Decis async def _get_token(self, deadline: float) -> Optional[str]: """Return a bearer token, or None when the deadline passed while waiting for another request's token. + A credential whose ``get_token`` is a coroutine function, or that implements ``AsyncTokenCredential``, is + asked on this event loop; any other credential is asked on a worker thread, once for concurrent requests. + :param float deadline: Absolute ``time.monotonic()`` instant by which the request must complete, not a duration or a wall-clock time; the caller derives it from the request timeout. :return: The bearer token, or None when the deadline passed while this request waited for a token that @@ -724,14 +727,17 @@ async def _get_token(self, deadline: float) -> Optional[str]: :rtype: Optional[str] """ credential = self._credential - token = _TOKENS.cached(credential, _MODEL_TOKEN_SCOPE) + token = _TOKENS.cached(credential, _DECISION_TOKEN_SCOPE) if token is None: # An acquisition that begins before a service rejects a token may return that token; the generation it # began under lets the cache refuse it. - generation = _TOKENS.generation(credential, _MODEL_TOKEN_SCOPE) - if inspect.iscoroutinefunction(credential.get_token): - token = await credential.get_token(_MODEL_TOKEN_SCOPE) - _TOKENS.store(credential, _MODEL_TOKEN_SCOPE, token, generation) + generation = _TOKENS.generation(credential, _DECISION_TOKEN_SCOPE) + if is_async_credential(credential): + # An asynchronous credential's token request may need this event loop. + token = credential.get_token(_DECISION_TOKEN_SCOPE) + if inspect.isawaitable(token): + token = await token + _TOKENS.store(credential, _DECISION_TOKEN_SCOPE, token, generation) else: abandoned = threading.Event() try: @@ -740,7 +746,7 @@ async def _get_token(self, deadline: float) -> Optional[str]: token = await asyncio.to_thread( _TOKENS.fetch, credential, - _MODEL_TOKEN_SCOPE, + _DECISION_TOKEN_SCOPE, abandoned=abandoned, deadline=deadline, describe_failure=_token_failure, @@ -751,7 +757,7 @@ async def _get_token(self, deadline: float) -> Optional[str]: return None if inspect.isawaitable(token): token = await token - _TOKENS.store(credential, _MODEL_TOKEN_SCOPE, token, generation) + _TOKENS.store(credential, _DECISION_TOKEN_SCOPE, token, generation) value = getattr(token, "token", None) if not isinstance(value, str): raise _failed("Could not acquire a token for the decision model.", ErrorBlame.USER_ERROR) @@ -827,7 +833,7 @@ async def _post(self, content: bytes, *, deadline: float, execution: DecisionExe if status != 200: if status == 401 and token is not None: # A rejected token is not reused. - _TOKENS.discard(self._credential, _MODEL_TOKEN_SCOPE, token) + _TOKENS.discard(self._credential, _DECISION_TOKEN_SCOPE, token) blame = ErrorBlame.USER_ERROR if status in (400, 401, 403, 404, 422) else ErrorBlame.SYSTEM_ERROR code = _provider_error_code(payload) kind = _provider_error_kind(status, payload) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_token_cache.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_token_cache.py index eca3b3ec3172..e553925100f2 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_token_cache.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_token_cache.py @@ -26,8 +26,9 @@ import threading import time import weakref -from typing import Any, Callable, Dict, Optional, Tuple +from typing import Any, Callable, Dict, NamedTuple, Optional, Tuple +from azure.core.credentials_async import AsyncTokenCredential from azure.core.exceptions import ClientAuthenticationError from azure.ai.evaluation._exceptions import EvaluationException @@ -38,6 +39,17 @@ TOKEN_WAIT_INTERVAL_SECONDS = 0.05 +def is_async_credential(credential: Any) -> bool: + """Whether a credential's token is requested on the caller's event loop rather than on a worker thread. + + That is a credential whose ``get_token`` is a coroutine function, or one that implements the asynchronous + credential protocol: its ``get_token`` may return an awaitable that needs the loop it was called on. + """ + return inspect.iscoroutinefunction(getattr(credential, "get_token", None)) or isinstance( + credential, AsyncTokenCredential + ) + + def _is_fresh(token: Any) -> bool: expires_on = getattr(token, "expires_on", None) if isinstance(expires_on, bool) or not isinstance(expires_on, (int, float)): @@ -69,16 +81,25 @@ def describe_credential_failure(error: BaseException) -> Optional[Dict[str, Any] return None +class _SharedFailure(NamedTuple): + """A failed token request's description and the attempt count after it, kept for requests of the same kind.""" + + attempt: int + description: Dict[str, Any] + + class _CredentialTokens: - """One credential's cached token for one scope, the lock for its token requests, the outcome of its latest token - request and the count of rejections that invalidates acquisitions begun before them.""" + """One credential's cached token for one scope, the lock for its token requests, the latest failure that each + kind of request described, and the count of rejections that invalidates acquisitions begun before them.""" def __init__(self, reference: Any) -> None: self.reference = reference self.token: Any = None self.lock = threading.Lock() self.attempts = 0 - self.failure: Optional[Dict[str, Any]] = None + # Keyed by the failure formatter and exception type of the requests that share the failure, so requests that + # describe or raise failures differently never raise each other's. Cleared when a token request succeeds. + self.failures: Dict[Tuple[Callable[..., Any], Callable[..., BaseException]], _SharedFailure] = {} # The latest token request's token when it was not cached, because a rejection kept it out or because of its # expiry, for the requests that waited on that request. It is never cached. self.unstored: Any = None @@ -91,10 +112,11 @@ class CredentialTokenCache: """Share each caller credential's token for a scope across requests until shortly before it expires. Concurrent requests that share a synchronous credential wait for one token request. A caller can also ask that - requests which waited on a failed token request share its failure instead of each requesting a token again. A - token a service rejected is discarded; the credential's next token is cached again, even when it is the same - token, while an acquisition that began before the rejection is never stored. Requests that waited on an - acquisition whose token was not cached use that token instead of each requesting another one. + requests which waited on a failed token request share its failure instead of each requesting a token again; only + requests that describe and raise failures the same way share one. A token a service rejected is discarded; the + credential's next token is cached again, even when it is the same token, while an acquisition that began before + the rejection is never stored. Requests that waited on an acquisition whose token was not cached use that token + instead of each requesting another one. """ def __init__(self) -> None: @@ -174,10 +196,13 @@ def fetch( """Request a synchronous credential's token once for concurrent requests. Returns None, without requesting a token, when the request was abandoned or its monotonic ``deadline`` passed - while it waited for another request's token. With ``describe_failure``, requests that waited on a failed - request raise ``failure_type`` built from its description instead of each requesting a token again; a None - description is not shared. The description must not keep the error, whose traceback references the - credential. Requests that waited on a request whose token was not cached use that token. + while it waited for another request's token. With ``describe_failure``, a request that waited on a failed + request of its own kind, one that passed the same ``describe_failure`` and ``failure_type``, raises + ``failure_type`` built from that request's description instead of requesting a token again. A None + description is not shared, and a request never raises a failure that another kind of request described, so + each keeps its own error type and classification; with no failure of its own kind, it makes its own attempt. + The description must not keep the error, whose traceback references the credential. Requests that waited on a + request whose token was not cached use that token. """ entry = self._entry(credential, scope, create=True) if entry is None: @@ -203,12 +228,13 @@ def fetch( or (deadline is not None and time.monotonic() >= deadline) ): return token + kind = (describe_failure, failure_type) with self._guard: waited = entry.attempts != queued_after - failure = entry.failure if waited and describe_failure is not None else None + failure = entry.failures.get(kind) if waited and describe_failure is not None else None unstored = entry.unstored if waited else None - if failure is not None: - raise failure_type(**failure) + if failure is not None and failure.attempt > queued_after: + raise failure_type(**failure.description) if _has_value(unstored): # Issued for the request this one waited on and not rejected since it was shared, so use it rather # than request another token. @@ -218,9 +244,13 @@ def fetch( try: token = credential.get_token(scope) except Exception as error: + description = describe_failure(error) if describe_failure is not None else None with self._guard: entry.attempts += 1 - entry.failure = describe_failure(error) if describe_failure is not None else None + if description is not None: + entry.failures[kind] = _SharedFailure(entry.attempts, description) + else: + entry.failures.pop(kind, None) entry.unstored = None raise # Decide whether the token is cached and publish the outcome together, so a rejection cannot land between @@ -228,7 +258,7 @@ def fetch( with self._guard: stored = inspect.isawaitable(token) or self.store(credential, scope, token, generation) entry.attempts += 1 - entry.failure = None + entry.failures.clear() entry.unstored = token if not stored and _has_value(token) else None return token finally: diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py index ad944656b7a6..6020365b0bca 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py @@ -43,7 +43,7 @@ from azure.ai.evaluation._constants import DEFAULT_MAX_COMPLETION_TOKENS_REASONING_MODELS from azure.ai.evaluation._legacy._common._logging import get_logger from azure.ai.evaluation._legacy._common._async_token_provider import AsyncAzureTokenProvider -from azure.ai.evaluation._legacy._common._token_cache import TOKENS, describe_credential_failure +from azure.ai.evaluation._legacy._common._token_cache import TOKENS, describe_credential_failure, is_async_credential from azure.ai.evaluation._user_agent import UserAgentSingleton PROMPTY_EXTENSION: Final[str] = ".prompty" @@ -455,11 +455,13 @@ class _ReusedTokenProvider: """Supply a credential's token to a chat client, reusing it until shortly before it expires. Concurrent requests that share a synchronous credential wait for one token request, on a worker thread so the - event loop is not blocked. The default token provider is represented by the synchronous credential it creates. - The request that made a failed token request raises the credential's error. When that failure was an - authentication failure, requests that waited on it raise a ``ClientAuthenticationError`` with the same message - instead of each requesting a token again; after any other failure, each waiting request makes its own attempt. A - token the service rejected is discarded, and an acquisition that began before the rejection is not stored. + event loop is not blocked. An asynchronous credential, one whose ``get_token`` is a coroutine function or that + implements ``AsyncTokenCredential``, is asked on the caller's event loop instead. The default token provider is + represented by the synchronous credential it creates. The request that made a failed token request raises the + credential's error. When that failure was an authentication failure, requests that waited on it raise a + ``ClientAuthenticationError`` with the same message instead of each requesting a token again; after any other + failure, each waiting request makes its own attempt. A token the service rejected is discarded, and an acquisition + that began before the rejection is not stored. """ def __init__(self, credential: Union[TokenCredential, AsyncTokenCredential], scope: str) -> None: @@ -479,8 +481,11 @@ async def __call__(self) -> str: # An acquisition that begins before a service rejects a token may return that token; the generation it # began under lets the cache refuse it. generation = TOKENS.generation(credential, self._scope) - if inspect.iscoroutinefunction(credential.get_token): - token = await credential.get_token(self._scope) + if is_async_credential(credential): + # An asynchronous credential's token request may need this event loop. + token = credential.get_token(self._scope) + if inspect.isawaitable(token): + token = await token TOKENS.store(credential, self._scope, token, generation) else: abandoned = threading.Event() diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_builtin_decision_evaluators.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_builtin_decision_evaluators.py index c8910abf4c10..5c1e82fec93e 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_builtin_decision_evaluators.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_builtin_decision_evaluators.py @@ -804,7 +804,7 @@ def test_qa_passes_the_credential_and_decision_controls_to_its_members(service): assert runner._max_retries == 1 and runner._cancellation_requested is _never_cancelled # Every member's request carries the forwarded credential's token, however many token requests that took. assert service.authorizations == ["Bearer decision-token"] * len(QA_KEYS) - assert credential.scopes and set(credential.scopes) == {("https://ai.azure.com/.default",)} + assert credential.scopes and set(credential.scopes) == {("https://cognitiveservices.azure.com/.default",)} assert [result[f"{key}_status"] for key in QA_KEYS] == ["completed"] * len(QA_KEYS) assert result["f1_score"] == pytest.approx(5 / 6) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index fd8d977b6d12..94f43352019a 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -154,7 +154,7 @@ def watch_token_queue(credential, requests): Each request reads the cache's attempt count before it queues, so a token request released after this event completes while every request is queued, whatever the thread scheduling. """ - entry = _model_runners._TOKENS._entry(credential, _model_runners._MODEL_TOKEN_SCOPE, create=True) + entry = _model_runners._TOKENS._entry(credential, _model_runners._DECISION_TOKEN_SCOPE, create=True) lock, queued, all_queued = entry.lock, set(), threading.Event() class QueueWatchingLock: @@ -339,7 +339,7 @@ def test_credential_acquires_a_scoped_bearer_token(self, client, asynchronous): credential.get_token = AsyncMock(return_value=token) if asynchronous else Mock(return_value=token) config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} run(DecisionModelRunner(config, credential)) - credential.get_token.assert_called_once_with("https://ai.azure.com/.default") + credential.get_token.assert_called_once_with("https://cognitiveservices.azure.com/.default") headers = client.calls[0]["headers"] assert headers["Authorization"] == "Bearer bearer-value" and "api-key" not in headers @@ -702,7 +702,7 @@ def test_fresh_token_is_reused_across_requests_and_runners(self, client, asynchr first, second = DecisionModelRunner(self.CONFIG, credential), DecisionModelRunner(self.CONFIG, credential) for runner in (first, second, first): run(runner) - credential.get_token.assert_called_once_with("https://ai.azure.com/.default") + credential.get_token.assert_called_once_with("https://cognitiveservices.azure.com/.default") assert self.authorizations(client) == ["Bearer shared-token"] * 3 @pytest.mark.parametrize("lifetime", [None, 60], ids=["no-expiry", "near-expiry"]) @@ -767,16 +767,15 @@ def get_token(*_, **__): request.result(2 * WAIT_SECONDS) assert len(requested) == 1 assert self.authorizations(client) == ["Bearer uncached"] * 6 - assert _model_runners._TOKENS.cached(credential, _model_runners._MODEL_TOKEN_SCOPE) is None + assert _model_runners._TOKENS.cached(credential, _model_runners._DECISION_TOKEN_SCOPE) is None - def test_requests_the_token_scope_chat_models_use(self, client): + def test_decision_tokens_are_kept_apart_from_chat_tokens(self, client): client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - credential = Mock(get_token=Mock(return_value=self.access_token("shared"))) + credential = Mock(get_token=Mock(return_value=self.access_token("decision"))) run(DecisionModelRunner(self.CONFIG, credential)) - credential.get_token.assert_called_once_with("https://ai.azure.com/.default") - assert _model_runners._MODEL_TOKEN_SCOPE == TokenScope.COGNITIVE_SERVICES_MANAGEMENT - # The chat path keys the cache with the enum member; the entry is the same one. - assert _model_runners._TOKENS.cached(credential, TokenScope.COGNITIVE_SERVICES_MANAGEMENT).token == "shared" + credential.get_token.assert_called_once_with("https://cognitiveservices.azure.com/.default") + # Chat requests key the cache with their own scope, so they never reuse a Decision token. + assert _model_runners._TOKENS.cached(credential, TokenScope.COGNITIVE_SERVICES_MANAGEMENT) is None def test_rejected_token_is_not_reused(self, client): client.response = FakeResponse(status_code=401, body=b"{}") @@ -809,7 +808,7 @@ def test_a_token_the_credential_reissues_after_a_rejection_is_cached_again(self, runner = DecisionModelRunner(self.CONFIG, credential) with pytest.raises(EvaluationException, match="HTTP 401"): run(runner) - assert _model_runners._TOKENS.cached(credential, _model_runners._MODEL_TOKEN_SCOPE) is None + assert _model_runners._TOKENS.cached(credential, _model_runners._DECISION_TOKEN_SCOPE) is None # The service accepts the token now, for example after a role assignment propagated. The credential # re-issues the same token: it is requested once more, then reused again instead of once per request. client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) @@ -817,10 +816,10 @@ def test_a_token_the_credential_reissues_after_a_rejection_is_cached_again(self, run(runner) assert credential.get_token.call_count == 2 assert self.authorizations(client) == ["Bearer same"] * 4 - assert _model_runners._TOKENS.cached(credential, _model_runners._MODEL_TOKEN_SCOPE).token == "same" + assert _model_runners._TOKENS.cached(credential, _model_runners._DECISION_TOKEN_SCOPE).token == "same" def test_an_acquisition_that_began_before_two_rejections_is_not_stored(self): - scope = _model_runners._MODEL_TOKEN_SCOPE + scope = _model_runners._DECISION_TOKEN_SCOPE async def scenario(): entered, gate = asyncio.Event(), asyncio.Event() @@ -1063,7 +1062,7 @@ def get_token(*_, **__): credential = Mock(get_token=Mock(side_effect=get_token)) all_queued = watch_token_queue(credential, 4) - scope = _model_runners._MODEL_TOKEN_SCOPE + scope = _model_runners._DECISION_TOKEN_SCOPE with ThreadPoolExecutor(max_workers=4) as pool: try: requests = [ @@ -1089,7 +1088,7 @@ def test_a_token_rejected_before_its_request_resumes_is_not_stored_again(self, c client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) credential = Mock(get_token=Mock(return_value=self.access_token("shared"))) fetch = _model_runners._TOKENS.fetch - scope = _model_runners._MODEL_TOKEN_SCOPE + scope = _model_runners._DECISION_TOKEN_SCOPE def fetch_then_reject(*args, **kwargs): token = fetch(*args, **kwargs) @@ -1171,7 +1170,7 @@ def get_token(self, *_, **__): credential = Credential() run(DecisionModelRunner(self.CONFIG, credential)) - assert _model_runners._TOKENS.cached(credential, _model_runners._MODEL_TOKEN_SCOPE) is token + assert _model_runners._TOKENS.cached(credential, _model_runners._DECISION_TOKEN_SCOPE) is token reference = weakref.ref(credential) del credential gc.collect() @@ -1207,7 +1206,7 @@ def test_a_credential_that_returns_no_usable_token_is_a_user_error(self, client, def test_a_token_wait_ends_at_the_deadline_without_requesting_a_token(self): credential = Mock(get_token=Mock(return_value=self.access_token("unused"))) - scope = _model_runners._MODEL_TOKEN_SCOPE + scope = _model_runners._DECISION_TOKEN_SCOPE entry = _model_runners._TOKENS._entry(credential, scope, create=True) runner = DecisionModelRunner(self.CONFIG, credential) # Another request holds the credential's token request throughout. @@ -1222,7 +1221,7 @@ def test_a_token_wait_ends_at_the_deadline_without_requesting_a_token(self): def test_a_credential_without_weak_references_is_not_asked_after_the_deadline(self): requested = [] credential = SimpleNamespace(get_token=lambda *scopes, **_: requested.append(scopes)) - scope = _model_runners._MODEL_TOKEN_SCOPE + scope = _model_runners._DECISION_TOKEN_SCOPE assert _model_runners._TOKENS.fetch(credential, scope, deadline=time.monotonic() - 1) is None assert requested == [] diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_token_reuse.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_token_reuse.py index dc2bac8869e9..a10da5861766 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_token_reuse.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_token_reuse.py @@ -24,12 +24,12 @@ from azure.ai.evaluation._constants import TokenScope from azure.ai.evaluation._exceptions import EvaluationException from azure.ai.evaluation._legacy._common._async_token_provider import AsyncAzureTokenProvider -from azure.ai.evaluation._legacy._common._token_cache import TOKENS +from azure.ai.evaluation._legacy._common._token_cache import TOKENS, describe_credential_failure, is_async_credential from azure.ai.evaluation._legacy.prompty import AsyncPrompty from azure.ai.evaluation._legacy.prompty._exceptions import WrappedOpenAIError CHAT_SCOPE = TokenScope.COGNITIVE_SERVICES_MANAGEMENT -DECISION_SCOPE = _model_runners._MODEL_TOKEN_SCOPE +DECISION_SCOPE = _model_runners._DECISION_TOKEN_SCOPE COHERENCE_PROMPTY = Path(coherence.__file__).parent / "coherence.prompty" CHAT_MODEL = construct_prompty_model_config( validate_model_config({"azure_endpoint": "https://example.openai.azure.com", "azure_deployment": "judge"}), @@ -201,13 +201,13 @@ async def scenario(): join_all([leader]) assert len(credential.scopes) == 1 - def test_chat_requests_reuse_a_token_a_decision_request_acquired(self): - credential = Credential(access_token("unused")) + def test_chat_and_decision_tokens_are_kept_apart(self): + credential = Credential(access_token("chat")) TOKENS.store(credential, DECISION_SCOPE, access_token("decision")) - assert request_token(credential) == "decision" - assert credential.scopes == [] - assert TOKENS.cached(credential, CHAT_SCOPE).token == "decision" + assert request_token(credential) == "chat" + assert credential.scopes == [(CHAT_SCOPE,)] + assert TOKENS.cached(credential, DECISION_SCOPE).token == "decision" def test_a_failed_token_request_raises_the_credential_error_and_is_retried(self): credential = Credential(ClientAuthenticationError("Please run az login."), access_token("recovered")) @@ -646,3 +646,209 @@ async def get_token(self, *_, **__): assert TOKENS.cached(credential, CHAT_SCOPE).token == "token-B" asyncio.run(scenario()) + + +DECISION_CONFIG = { + "type": "decision", + "azure_endpoint": "https://example.services.ai.azure.com", + "azure_deployment": "Decision-1", +} + + +def request_decision_token(credential): + runner = _model_runners.DecisionModelRunner(DECISION_CONFIG, credential) + return asyncio.run(runner._get_token(time.monotonic() + 2 * WAIT_SECONDS)) + + +class GatedCredential(Credential): + """A ``Credential`` whose token requests wait until the test sets ``release``, and that reports the first one.""" + + def __init__(self, *results): + self.entered, self.release = threading.Event(), threading.Event() + super().__init__(*results, gate=self.release) + + def get_token(self, *scopes, **kwargs): + self.entered.set() + return super().get_token(*scopes, **kwargs) + + +def queue_behind(credential, leader, *waiters): + """Return the outcomes of ``leader``'s token request and of each of ``waiters``, all queued behind it.""" + requests = [leader, *waiters] + all_queued = watch_token_queue(credential, len(requests)) + outcomes = [None] * len(requests) + + def request(index): + try: + outcomes[index] = requests[index](credential) + except Exception as error: # pylint: disable=broad-except + outcomes[index] = error + + threads = [threading.Thread(target=request, args=(index,)) for index in range(len(requests))] + try: + threads[0].start() + assert credential.entered.wait(WAIT_SECONDS) + for thread in threads[1:]: + thread.start() + assert all_queued.wait(WAIT_SECONDS) + finally: + credential.release.set() + join_all(threads) + return outcomes + + +def fetch_as(describe_failure, failure_type): + """Return a request for the credential's chat-scope token that describes and raises failures its own way.""" + + def request(credential): + return TOKENS.fetch(credential, CHAT_SCOPE, describe_failure=describe_failure, failure_type=failure_type).token + + return request + + +# Two kinds of request that describe and raise failures differently, as chat and Decision requests do. +AUTHENTICATION_KIND = fetch_as(describe_credential_failure, ClientAuthenticationError) +EVALUATION_KIND = fetch_as(_model_runners._token_failure, EvaluationException) +EVALUATION_MESSAGE = "Could not acquire a token for the decision model." + + +@pytest.mark.unittest +class TestFailuresAreSharedWithinOneKindOfRequest: + """Requests that describe failures differently share a credential's tokens, but not each other's failures.""" + + @pytest.mark.parametrize("leader", ["authentication", "evaluation"]) + def test_both_kinds_of_request_share_one_token_request(self, leader): + kinds = {"authentication": AUTHENTICATION_KIND, "evaluation": EVALUATION_KIND} + waiter = "evaluation" if leader == "authentication" else "authentication" + credential = GatedCredential(access_token("shared")) + + assert queue_behind(credential, kinds[leader], kinds[waiter]) == ["shared", "shared"] + assert len(credential.scopes) == 1 + + @pytest.mark.parametrize("retried", ["recovered", "failed again"]) + def test_an_evaluation_failure_is_not_replayed_to_another_kind_of_request(self, retried): + retry = access_token("recovered") if retried == "recovered" else ClientAuthenticationError("Run az login.") + credential = GatedCredential(ClientAuthenticationError("Please run az login."), retry) + + evaluation, authentication = queue_behind(credential, EVALUATION_KIND, AUTHENTICATION_KIND) + + assert isinstance(evaluation, ClientAuthenticationError) + # The other request asks the credential itself rather than raising the evaluation description as its own + # authentication error. + if retried == "recovered": + assert authentication == "recovered" + else: + assert isinstance(authentication, ClientAuthenticationError) and str(authentication) == "Run az login." + assert len(credential.scopes) == 2 + + @pytest.mark.parametrize("retried", ["recovered", "failed again"]) + def test_an_authentication_failure_is_not_replayed_to_another_kind_of_request(self, retried): + retry = access_token("recovered") if retried == "recovered" else ClientAuthenticationError("Run az login.") + credential = GatedCredential(ClientAuthenticationError("Please run az login."), retry) + + authentication, evaluation = queue_behind(credential, AUTHENTICATION_KIND, EVALUATION_KIND) + + assert isinstance(authentication, ClientAuthenticationError) and "az login" in str(authentication) + # The other request asks the credential itself rather than raising an unclassified EvaluationException. + if retried == "recovered": + assert evaluation == "recovered" + else: + assert isinstance(evaluation, ClientAuthenticationError) and str(evaluation) == "Run az login." + assert len(credential.scopes) == 2 + + def test_evaluation_failures_are_shared_whichever_request_wakes_first(self): + credential = GatedCredential(ClientAuthenticationError("Please run az login.")) + + evaluation, authentication, other_evaluation = queue_behind( + credential, EVALUATION_KIND, AUTHENTICATION_KIND, EVALUATION_KIND + ) + + assert isinstance(evaluation, ClientAuthenticationError) + # The other kind of request makes its own attempt and raises the credential's error. The request of the same + # kind shares the first failure even when the other kind's attempt ends first. + assert isinstance(authentication, ClientAuthenticationError) and str(authentication) == "Please run az login." + assert isinstance(other_evaluation, EvaluationException) and EVALUATION_MESSAGE in str(other_evaluation) + assert len(credential.scopes) == 2 + + def test_authentication_failures_are_shared_whichever_request_wakes_first(self): + credential = GatedCredential(ClientAuthenticationError("Please run az login.")) + + authentication, evaluation, other_authentication = queue_behind( + credential, AUTHENTICATION_KIND, EVALUATION_KIND, AUTHENTICATION_KIND + ) + + assert isinstance(authentication, ClientAuthenticationError) + assert isinstance(evaluation, ClientAuthenticationError) and not isinstance(evaluation, EvaluationException) + assert isinstance(other_authentication, ClientAuthenticationError) + assert str(other_authentication) == "Please run az login." + assert len(credential.scopes) == 2 + + def test_a_successful_token_request_ends_sharing_of_earlier_failures(self): + credential = Credential(ClientAuthenticationError("Please run az login."), access_token("recovered")) + + with pytest.raises(ClientAuthenticationError): + EVALUATION_KIND(credential) + entry = TOKENS._entry(credential, CHAT_SCOPE) + assert list(entry.failures) == [(_model_runners._token_failure, EvaluationException)] + + assert AUTHENTICATION_KIND(credential) == "recovered" + # A request that waited on the failure but wakes after this success gets the token, not the earlier failure. + assert entry.failures == {} + + +class LoopBoundAsyncCredential: + """An asynchronous credential whose ``get_token`` is a plain function that needs the caller's event loop.""" + + def __init__(self): + self.calls = 0 + + def get_token(self, *_, **__): + self.calls += 1 + loop = asyncio.get_running_loop() + + async def request(): + assert asyncio.get_running_loop() is loop + return access_token("loop-bound") + + return request() + + async def close(self): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + pass + + +@pytest.mark.unittest +class TestAsynchronousCredentialRequests: + @pytest.mark.parametrize("kind", ["chat", "decision"]) + def test_an_asynchronous_credential_requests_its_token_on_the_callers_event_loop(self, kind): + make_request = {"chat": request_token, "decision": request_decision_token}[kind] + credential = LoopBoundAsyncCredential() + + assert [make_request(credential) for _ in range(2)] == ["loop-bound"] * 2 + assert credential.calls == 1 + + def test_synchronous_identity_credentials_request_tokens_on_a_worker_thread(self): + from azure.identity import ( + AzureCliCredential, + ClientSecretCredential, + DefaultAzureCredential, + ManagedIdentityCredential, + ) + from azure.identity.aio import AzureCliCredential as AsyncAzureCliCredential + + synchronous = [ + AzureCliCredential(), + ClientSecretCredential("tenant", "client", "secret"), + DefaultAzureCredential(exclude_shared_token_cache_credential=True), + ManagedIdentityCredential(), + Credential(access_token("sync")), + ] + + assert not any(is_async_credential(credential) for credential in synchronous) + assert is_async_credential(AsyncAzureCliCredential()) + assert is_async_credential(LoopBoundAsyncCredential()) From 84df6b02ebeaeda7cf765b9fffa11963b76cb0b2 Mon Sep 17 00:00:00 2001 From: Ahmad Qardahji Date: Tue, 3 Nov 2026 11:51:03 -0800 Subject: [PATCH 05/14] Format the Decision error-kind patterns and tests with black The package enforces black, and these lines were not formatted. No code changes. --- .../_evaluators/_common/_model_runners.py | 14 ++++++--- .../unittests/test_decision_model_support.py | 29 ++++++++++++++----- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py index 8150c458e74a..b14eae8cccc2 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py @@ -70,10 +70,16 @@ def value(self) -> str: # reported as a bounded ``decision_model.error_kind``; any other sentence is ignored. Only the sentence's first # characters are read, which hold every opening, so a long question identifier cannot hide the reason. _PROVIDER_DETAIL_KINDS = ( - (re.compile(r"invalid model request: Question prompt exceeds the \d+-token context limit\b"), - "context_window_exceeded"), - (re.compile(r"invalid TypeSafe request: request is about \d+ tokens; the limit for the state plus all questions is \d+"), - "request_token_limit_exceeded"), + ( + re.compile(r"invalid model request: Question prompt exceeds the \d+-token context limit\b"), + "context_window_exceeded", + ), + ( + re.compile( + r"invalid TypeSafe request: request is about \d+ tokens; the limit for the state plus all questions is \d+" + ), + "request_token_limit_exceeded", + ), (re.compile(r"invalid TypeSafe request: request has \d+ questions; the limit is \d+"), "question_limit_exceeded"), (re.compile(r"invalid TypeSafe request: invalid [a-z_]+ for question \""), "invalid_question"), ) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index 94f43352019a..fbd5c3484eab 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -508,14 +508,25 @@ def test_provider_http_error_code_preserves_classification_and_accepted_metadata "request_token_limit_exceeded", ), ("invalid TypeSafe request: request has 256 questions; the limit is 255", "question_limit_exceeded"), - ('invalid TypeSafe request: invalid instructions for question "private-key": value must not be empty', - "invalid_question"), - ('invalid TypeSafe request: invalid criteria for question "q": value must not be empty', "invalid_question"), + ( + 'invalid TypeSafe request: invalid instructions for question "private-key": value must not be empty', + "invalid_question", + ), + ( + 'invalid TypeSafe request: invalid criteria for question "q": value must not be empty', + "invalid_question", + ), ("invalid TypeSafe request: private-message test-key-value", None), - ("invalid model request: Question prompt exceeds the 32768-token context limit " + "x" * 5000, - "context_window_exceeded"), - ('invalid TypeSafe request: invalid instructions for question "' + "q" * 2000 + '": value must not be empty', - "invalid_question"), + ( + "invalid model request: Question prompt exceeds the 32768-token context limit " + "x" * 5000, + "context_window_exceeded", + ), + ( + 'invalid TypeSafe request: invalid instructions for question "' + + "q" * 2000 + + '": value must not be empty', + "invalid_question", + ), ("x" * 1100 + "invalid model request: Question prompt exceeds the 32768-token context limit", None), ("", None), ], @@ -1005,7 +1016,9 @@ def request(name): assert all("private-detail" not in str(error) for error in errors.values()) credential.get_token.side_effect = lambda *_, **__: self.access_token("recovered") run(DecisionModelRunner(self.CONFIG, credential)) - assert credential.get_token.call_count == token_requests + 1 and self.authorizations(client) == ["Bearer recovered"] + assert credential.get_token.call_count == token_requests + 1 and self.authorizations(client) == [ + "Bearer recovered" + ] def test_requests_waiting_on_a_transient_token_failure_recover_with_one_more_request(self, client): client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) From 44097f4659ee44a217ebef35cadb7b11dd888ec8 Mon Sep 17 00:00:00 2001 From: Ahmad Qardahji Date: Tue, 3 Nov 2026 12:45:59 -0800 Subject: [PATCH 06/14] Format the Decision batch persistence test with black The Analyze job's black check failed on this file. No code changes. --- .../tests/unittests/test_decision_batch_persistence.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py index e7581365e537..49020631c5eb 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py @@ -238,9 +238,7 @@ def test_failed_metadata_rejects_untyped_and_unrecognized_provenance(self): ) def test_failed_metadata_keeps_only_a_known_error_kind(self, kind, kept): error = EvaluationException("synthetic failure", properties={"decision_model": {"error_kind": kind}}) - assert _decision_error_metadata(error)["properties"] == { - "decision_model": {"error_kind": kind} if kept else {} - } + assert _decision_error_metadata(error)["properties"] == {"decision_model": {"error_kind": kind} if kept else {}} @pytest.mark.parametrize( "detail,max_request_bytes,kind,message", From d960c7978c91d8e277fa2995fe268581335ed2c7 Mon Sep 17 00:00:00 2001 From: Ahmad Qardahji Date: Tue, 3 Nov 2026 12:52:36 -0800 Subject: [PATCH 07/14] Ask credentials that also implement the synchronous context manager protocol on a worker thread A synchronous credential whose class also implements the asynchronous context manager protocol matched the structural AsyncTokenCredential check and was asked on the event loop, which blocked the loop, lost the shared token request and broke a get_token that runs its own event loop. A credential that implements the synchronous context manager protocol is again asked on a worker thread. A plain get_token that needs the caller's running loop must belong to an AsyncTokenCredential, as the CHANGELOG and docstrings now state, and fetch documents that its failure formatter and exception type key the shared failures and must be module-level. --- .../azure-ai-evaluation/CHANGELOG.md | 2 +- .../_evaluators/_common/_model_runners.py | 5 ++- .../_legacy/_common/_token_cache.py | 15 +++++--- .../ai/evaluation/_legacy/prompty/_prompty.py | 13 ++++--- .../unittests/test_prompty_token_reuse.py | 37 +++++++++++++++++++ 5 files changed, 58 insertions(+), 14 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index 6bb455b3f68a..58e1209dfa18 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -12,7 +12,7 @@ ### Bugs Fixed -- Decision requests, and Azure OpenAI chat model requests that authenticate with Microsoft Entra ID, reuse a credential's token until five minutes before it expires, as azure-core's bearer token policy does, instead of requesting a token for every request. Concurrent evaluations that share a synchronous credential wait for one token request, without blocking their event loops. A credential whose `get_token` is a coroutine function, or that implements `AsyncTokenCredential`, is asked for its token on the caller's event loop, and any other credential on a worker thread. With developer credentials such as `AzureCliCredential`, which start a process per request, concurrent evaluations were slow and could fail. Requests that waited on a token request that failed authentication share its failure instead of each requesting a token again; after any other failure, such as a token service outage, each waiting request makes its own attempt, so a transient failure fails only the request that hit it. Decision and chat requests behave the same way. A token rejected with HTTP 401 is dropped and requested again; a token whose request began before the rejection is not kept for reuse, and requests that waited on it use it rather than each requesting another one. The default credential that a chat evaluator creates when no `credential` is passed is shared the same way. Tokens are kept per scope, and the cache never keeps a credential alive. A token without an expiry, or that expires within five minutes, is not kept, but the requests that waited for it use it. +- Decision requests, and Azure OpenAI chat model requests that authenticate with Microsoft Entra ID, reuse a credential's token until five minutes before it expires, as azure-core's bearer token policy does, instead of requesting a token for every request. Concurrent evaluations that share a synchronous credential wait for one token request, without blocking their event loops. A credential whose `get_token` is a coroutine function, or that implements `AsyncTokenCredential` but not the synchronous context manager protocol, is asked for its token on the caller's event loop, and any other credential on a worker thread, so a plain `get_token` that needs the running event loop must belong to an `AsyncTokenCredential`. With developer credentials such as `AzureCliCredential`, which start a process per request, concurrent evaluations were slow and could fail. Requests that waited on a token request that failed authentication share its failure instead of each requesting a token again; after any other failure, such as a token service outage, each waiting request makes its own attempt, so a transient failure fails only the request that hit it. Decision and chat requests behave the same way. A token rejected with HTTP 401 is dropped and requested again; a token whose request began before the rejection is not kept for reuse, and requests that waited on it use it rather than each requesting another one. The default credential that a chat evaluator creates when no `credential` is passed is shared the same way. Tokens are kept per scope, and the cache never keeps a credential alive. A token without an expiry, or that expires within five minutes, is not kept, but the requests that waited for it use it. - `_ToolCallSuccessEvaluator` no longer scores successful tool calls 0 when a chat judge model, such as gpt-4.1 or gpt-4o, puts `score` and `status` inside `properties`. The example outputs in its prompt are now valid JSON, and `score`, `status` and `reason` are read from `properties` when the output has none at the top level. A missing score is no longer reported as 0, and any score other than 0 or 1 raises a system-classified `EvaluationException`, as `_ToolInputAccuracyEvaluator` does. - `_ToolCallSuccessEvaluator` reports a list of messages that holds no tool calls, such as one that contains only text or only MCP approval messages, as not applicable without calling the chat or Decision judge model. Strings, text that contains `[TOOL_CALL]` or `[TOOL_RESULT]`, and messages with any other content are still sent to the judge model. diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py index b14eae8cccc2..ecc024ce873c 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py @@ -723,8 +723,9 @@ async def _await_request(self, content: bytes, deadline: float, execution: Decis async def _get_token(self, deadline: float) -> Optional[str]: """Return a bearer token, or None when the deadline passed while waiting for another request's token. - A credential whose ``get_token`` is a coroutine function, or that implements ``AsyncTokenCredential``, is - asked on this event loop; any other credential is asked on a worker thread, once for concurrent requests. + A credential whose ``get_token`` is a coroutine function, or that implements ``AsyncTokenCredential`` but not + the synchronous context manager protocol, is asked on this event loop; any other credential is asked on a + worker thread, once for concurrent requests. :param float deadline: Absolute ``time.monotonic()`` instant by which the request must complete, not a duration or a wall-clock time; the caller derives it from the request timeout. diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_token_cache.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_token_cache.py index e553925100f2..be74ab288085 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_token_cache.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_token_cache.py @@ -21,6 +21,7 @@ need the coordination here. """ +import contextlib import inspect import math import threading @@ -43,10 +44,12 @@ def is_async_credential(credential: Any) -> bool: """Whether a credential's token is requested on the caller's event loop rather than on a worker thread. That is a credential whose ``get_token`` is a coroutine function, or one that implements the asynchronous - credential protocol: its ``get_token`` may return an awaitable that needs the loop it was called on. + credential protocol but not the synchronous context manager protocol: its ``get_token`` may return an awaitable + that needs the loop it was called on. Any other credential is asked on a worker thread, so a plain ``get_token`` + that needs the caller's running loop must belong to an ``AsyncTokenCredential``. """ - return inspect.iscoroutinefunction(getattr(credential, "get_token", None)) or isinstance( - credential, AsyncTokenCredential + return inspect.iscoroutinefunction(getattr(credential, "get_token", None)) or ( + isinstance(credential, AsyncTokenCredential) and not isinstance(credential, contextlib.AbstractContextManager) ) @@ -201,8 +204,10 @@ def fetch( ``failure_type`` built from that request's description instead of requesting a token again. A None description is not shared, and a request never raises a failure that another kind of request described, so each keeps its own error type and classification; with no failure of its own kind, it makes its own attempt. - The description must not keep the error, whose traceback references the credential. Requests that waited on a - request whose token was not cached use that token. + The description must not keep the error, whose traceback references the credential, and ``describe_failure`` + and ``failure_type`` must be module-level functions or classes: they key the shared failures, so a bound method + or closure would keep what it references, such as the credential, alive. Requests that waited on a request + whose token was not cached use that token. """ entry = self._entry(credential, scope, create=True) if entry is None: diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py index 6020365b0bca..fa1ad1b89b1e 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py @@ -456,12 +456,13 @@ class _ReusedTokenProvider: Concurrent requests that share a synchronous credential wait for one token request, on a worker thread so the event loop is not blocked. An asynchronous credential, one whose ``get_token`` is a coroutine function or that - implements ``AsyncTokenCredential``, is asked on the caller's event loop instead. The default token provider is - represented by the synchronous credential it creates. The request that made a failed token request raises the - credential's error. When that failure was an authentication failure, requests that waited on it raise a - ``ClientAuthenticationError`` with the same message instead of each requesting a token again; after any other - failure, each waiting request makes its own attempt. A token the service rejected is discarded, and an acquisition - that began before the rejection is not stored. + implements ``AsyncTokenCredential`` but not the synchronous context manager protocol, is asked on the caller's + event loop instead; a plain ``get_token`` that needs the running loop must therefore belong to an + ``AsyncTokenCredential``. The default token provider is represented by the synchronous credential it creates. The + request that made a failed token request raises the credential's error. When that failure was an authentication + failure, requests that waited on it raise a ``ClientAuthenticationError`` with the same message instead of each + requesting a token again; after any other failure, each waiting request makes its own attempt. A token the service + rejected is discarded, and an acquisition that began before the rejection is not stored. """ def __init__(self, credential: Union[TokenCredential, AsyncTokenCredential], scope: str) -> None: diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_token_reuse.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_token_reuse.py index a10da5861766..ad568952ce2f 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_token_reuse.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_token_reuse.py @@ -11,6 +11,7 @@ import weakref from pathlib import Path from types import SimpleNamespace +from unittest.mock import MagicMock import httpx import pytest @@ -822,6 +823,25 @@ async def __aexit__(self, *_): pass +class DualProtocolCredential(GatedCredential): + """A synchronous credential that implements both the synchronous and the asynchronous context manager protocols.""" + + def close(self): + pass + + def __enter__(self): + return self + + def __exit__(self, *_): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + pass + + @pytest.mark.unittest class TestAsynchronousCredentialRequests: @pytest.mark.parametrize("kind", ["chat", "decision"]) @@ -832,6 +852,21 @@ def test_an_asynchronous_credential_requests_its_token_on_the_callers_event_loop assert [make_request(credential) for _ in range(2)] == ["loop-bound"] * 2 assert credential.calls == 1 + def test_a_synchronous_credential_with_both_context_manager_protocols_is_asked_on_a_worker_thread(self): + credential = DualProtocolCredential(access_token("dual")) + + # Both requests wait for one token request on a worker thread; on the event loop each would ask the credential. + assert queue_behind(credential, request_token, request_token) == ["dual", "dual"] + assert len(credential.scopes) == 1 + + def test_a_plain_get_token_outside_the_asynchronous_protocol_is_asked_on_a_worker_thread(self): + class MinimalWrapper: + def get_token(self, *_, **__): + return asyncio.ensure_future(asyncio.sleep(0, access_token("minimal"))) + + # A plain get_token that needs the running event loop must belong to an AsyncTokenCredential. + assert not is_async_credential(MinimalWrapper()) + def test_synchronous_identity_credentials_request_tokens_on_a_worker_thread(self): from azure.identity import ( AzureCliCredential, @@ -847,6 +882,8 @@ def test_synchronous_identity_credentials_request_tokens_on_a_worker_thread(self DefaultAzureCredential(exclude_shared_token_cache_credential=True), ManagedIdentityCredential(), Credential(access_token("sync")), + MagicMock(get_token=MagicMock(return_value=access_token("mock")), close=MagicMock()), + DualProtocolCredential(access_token("dual")), ] assert not any(is_async_credential(credential) for credential in synchronous) From 4d7a19343a25f0bbdbf1f8f00a8624b9810fbc74 Mon Sep 17 00:00:00 2001 From: Ahmad Qardahji Date: Tue, 3 Nov 2026 13:22:28 -0800 Subject: [PATCH 08/14] Move token reuse to its own pull request Token reuse for Decision and chat requests is not part of the Tool Call Success and Decision failure reason fixes, so it moves to a separate pull request. This restores the base _prompty.py and _async_token_provider.py, removes _token_cache.py and its tests, and returns DecisionModelRunner to requesting a token for each call, with the Cognitive Services scope as before. The error_kind reasons, the retry policy cleanup and the Tool Call Success fixes are unchanged. --- .../azure-ai-evaluation/CHANGELOG.md | 1 - .../_evaluators/_common/_model_runners.py | 91 +- .../_legacy/_common/_async_token_provider.py | 64 +- .../_legacy/_common/_token_cache.py | 273 ------ .../ai/evaluation/_legacy/prompty/_prompty.py | 107 +-- .../unittests/test_decision_model_support.py | 601 +----------- .../unittests/test_prompty_token_reuse.py | 891 ------------------ 7 files changed, 46 insertions(+), 1982 deletions(-) delete mode 100644 sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_token_cache.py delete mode 100644 sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_token_reuse.py diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index 58e1209dfa18..07af8fbbc849 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -12,7 +12,6 @@ ### Bugs Fixed -- Decision requests, and Azure OpenAI chat model requests that authenticate with Microsoft Entra ID, reuse a credential's token until five minutes before it expires, as azure-core's bearer token policy does, instead of requesting a token for every request. Concurrent evaluations that share a synchronous credential wait for one token request, without blocking their event loops. A credential whose `get_token` is a coroutine function, or that implements `AsyncTokenCredential` but not the synchronous context manager protocol, is asked for its token on the caller's event loop, and any other credential on a worker thread, so a plain `get_token` that needs the running event loop must belong to an `AsyncTokenCredential`. With developer credentials such as `AzureCliCredential`, which start a process per request, concurrent evaluations were slow and could fail. Requests that waited on a token request that failed authentication share its failure instead of each requesting a token again; after any other failure, such as a token service outage, each waiting request makes its own attempt, so a transient failure fails only the request that hit it. Decision and chat requests behave the same way. A token rejected with HTTP 401 is dropped and requested again; a token whose request began before the rejection is not kept for reuse, and requests that waited on it use it rather than each requesting another one. The default credential that a chat evaluator creates when no `credential` is passed is shared the same way. Tokens are kept per scope, and the cache never keeps a credential alive. A token without an expiry, or that expires within five minutes, is not kept, but the requests that waited for it use it. - `_ToolCallSuccessEvaluator` no longer scores successful tool calls 0 when a chat judge model, such as gpt-4.1 or gpt-4o, puts `score` and `status` inside `properties`. The example outputs in its prompt are now valid JSON, and `score`, `status` and `reason` are read from `properties` when the output has none at the top level. A missing score is no longer reported as 0, and any score other than 0 or 1 raises a system-classified `EvaluationException`, as `_ToolInputAccuracyEvaluator` does. - `_ToolCallSuccessEvaluator` reports a list of messages that holds no tool calls, such as one that contains only text or only MCP approval messages, as not applicable without calling the chat or Decision judge model. Strings, text that contains `[TOOL_CALL]` or `[TOOL_RESULT]`, and messages with any other content are still sent to the judge model. diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py index ecc024ce873c..b2035dda5695 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py @@ -16,17 +16,14 @@ import math import os import re -import threading import time from typing import Any, Callable, Dict, List, Optional from urllib.parse import urlsplit -from azure.core.exceptions import ClientAuthenticationError from azure.core.pipeline.policies import AsyncRedirectPolicy, AsyncRetryPolicy, CustomHookPolicy from azure.core.pipeline.transport import AioHttpTransport from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException -from azure.ai.evaluation._legacy._common._token_cache import TOKENS as _TOKENS, is_async_credential from ..._common.utils import construct_prompty_model_config, validate_model_config from ..._http_utils import get_async_http_client @@ -47,9 +44,6 @@ def value(self) -> str: DECISION_MODEL_TYPE = "decision" _DECISION_PATH = "/providers/microsoft/v1/systemone" -# Decision endpoints also accept the chat scope, https://ai.azure.com/.default, but the hosted evaluation service -# relies on this audience to call Decision with its exchanged or managed identity token rather than the caller's own -# token, so Decision requests keep this scope and their own cached token. _DECISION_TOKEN_SCOPE = "https://cognitiveservices.azure.com/.default" LOGGER = logging.getLogger(__name__) _PREFIX = "azure.ai.evaluation.decision." @@ -368,25 +362,6 @@ def _is_finite_number(value: Any) -> bool: return False -def _token_failure(error: BaseException) -> Optional[Dict[str, Any]]: - """Describe an authentication failure for the requests that waited on it, or return None for any other failure. - - As on the chat path, only an authentication failure is shared: any other failure, such as a token service outage, - is not, so each waiting request makes its own attempt and one transient failure fails only the request that hit - it. The description never keeps the error, whose traceback references the credential. - """ - if not isinstance(error, ClientAuthenticationError): - return None - message = "Could not acquire a token for the decision model." - return { - "message": message, - "internal_message": message, - "target": ErrorTarget.EVALUATE, - "category": ErrorCategory.FAILED_EXECUTION, - "blame": ErrorBlame.USER_ERROR, - } - - def _provider_error_code(payload: Any) -> Optional[str]: """Return the provider's nested ``error.code`` when it is a bounded identifier, otherwise None.""" provider_error = payload.get("error") if isinstance(payload, dict) else None @@ -720,72 +695,25 @@ async def _await_request(self, content: bytes, deadline: float, execution: Decis task.cancel() await asyncio.gather(task, return_exceptions=True) - async def _get_token(self, deadline: float) -> Optional[str]: - """Return a bearer token, or None when the deadline passed while waiting for another request's token. - - A credential whose ``get_token`` is a coroutine function, or that implements ``AsyncTokenCredential`` but not - the synchronous context manager protocol, is asked on this event loop; any other credential is asked on a - worker thread, once for concurrent requests. - - :param float deadline: Absolute ``time.monotonic()`` instant by which the request must complete, not a - duration or a wall-clock time; the caller derives it from the request timeout. - :return: The bearer token, or None when the deadline passed while this request waited for a token that - another request was acquiring. - :rtype: Optional[str] - """ - credential = self._credential - token = _TOKENS.cached(credential, _DECISION_TOKEN_SCOPE) - if token is None: - # An acquisition that begins before a service rejects a token may return that token; the generation it - # began under lets the cache refuse it. - generation = _TOKENS.generation(credential, _DECISION_TOKEN_SCOPE) - if is_async_credential(credential): - # An asynchronous credential's token request may need this event loop. - token = credential.get_token(_DECISION_TOKEN_SCOPE) - if inspect.isawaitable(token): - token = await token - _TOKENS.store(credential, _DECISION_TOKEN_SCOPE, token, generation) - else: - abandoned = threading.Event() - try: - # A request waiting for another request's token holds a default-executor thread while it waits; - # past the executor's size, further requests queue for a thread before they reach the cache. - token = await asyncio.to_thread( - _TOKENS.fetch, - credential, - _DECISION_TOKEN_SCOPE, - abandoned=abandoned, - deadline=deadline, - describe_failure=_token_failure, - ) - finally: - abandoned.set() - if token is None and time.monotonic() >= deadline: - return None - if inspect.isawaitable(token): - token = await token - _TOKENS.store(credential, _DECISION_TOKEN_SCOPE, token, generation) - value = getattr(token, "token", None) - if not isinstance(value, str): - raise _failed("Could not acquire a token for the decision model.", ErrorBlame.USER_ERROR) - return value + async def _get_token(self) -> str: + if inspect.iscoroutinefunction(self._credential.get_token): + return (await self._credential.get_token(_DECISION_TOKEN_SCOPE)).token + token = await asyncio.to_thread(self._credential.get_token, _DECISION_TOKEN_SCOPE) + if inspect.isawaitable(token): + token = await token + return token.token async def _post(self, content: bytes, *, deadline: float, execution: DecisionExecution) -> Any: headers = {"x-ms-useragent": self._user_agent, "Content-Type": "application/json"} - token = None if self._api_key: headers["api-key"] = self._api_key else: try: - token = await self._get_token(deadline) + headers["Authorization"] = f"Bearer {await self._get_token()}" except EvaluationException: raise except Exception: # pylint: disable=broad-except raise _failed("Could not acquire a token for the decision model.", ErrorBlame.USER_ERROR) from None - if token is None: - execution.outcome = "deadline" - raise _failed("The decision model request timed out.") - headers["Authorization"] = "Bearer " + token policies: Dict[str, Any] = { "custom_hook_policy": execution, @@ -838,9 +766,6 @@ async def _post(self, content: bytes, *, deadline: float, execution: DecisionExe raise _failed("The decision model returned invalid JSON.") from None payload = None if status != 200: - if status == 401 and token is not None: - # A rejected token is not reused. - _TOKENS.discard(self._credential, _DECISION_TOKEN_SCOPE, token) blame = ErrorBlame.USER_ERROR if status in (400, 401, 403, 404, 422) else ErrorBlame.SYSTEM_ERROR code = _provider_error_code(payload) kind = _provider_error_kind(status, payload) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_async_token_provider.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_async_token_provider.py index 71f76db8b7f3..4509604921c7 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_async_token_provider.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_async_token_provider.py @@ -4,7 +4,6 @@ import logging import os -import threading from typing import Any, AsyncContextManager, Optional from azure.core.credentials import AccessToken, TokenCredential @@ -28,13 +27,11 @@ def __init__(self, *, base_url: Optional[str] = None, **kwargs: Any) -> None: """Initialize the AsyncAzureTokenProvider.""" self._credential: Optional[TokenCredential] = None self._env_client: Optional[AzureEnvironmentClient] = AzureEnvironmentClient(base_url=base_url, **kwargs) - self._credential_lock = threading.Lock() async def close(self) -> None: - with self._credential_lock: - credential, client = self._credential, self._env_client - self._credential = None - self._env_client = None + credential, client = self._credential, self._env_client + self._credential = None + self._env_client = None try: close_credential = getattr(credential, "close", None) if close_credential is not None: @@ -43,44 +40,6 @@ async def close(self) -> None: if client is not None: await client.close() - async def get_credential(self) -> TokenCredential: - """Return the synchronous credential that supplies this provider's tokens, creating it on first use. - - Concurrent first requests all use the first credential created, so they can share its token. - - :return: The credential. - :rtype: ~azure.core.credentials.TokenCredential - """ - credential = self._credential - if credential is None: - created = await self._initialize_async(self._env_client) - with self._credential_lock: - closed = self._env_client is None - # A provider closed while the credential was created keeps no credential. - if self._credential is None and not closed: - self._credential = created - credential = self._credential - if credential is not created: - close_created = getattr(created, "close", None) - if close_created is not None: - close_created() - if closed: - raise EvaluationException( - f"{AsyncAzureTokenProvider.__name__} instance has already been closed.", - target=ErrorTarget.UNKNOWN, - category=ErrorCategory.INVALID_VALUE, - blame=ErrorBlame.USER_ERROR, - ) - - if credential is None: - raise EvaluationException( - f"{self.__class__.__name__} could not determine the credential to use.", - target=ErrorTarget.UNKNOWN, - category=ErrorCategory.INVALID_VALUE, - blame=ErrorBlame.SYSTEM_ERROR, - ) - return credential - async def get_token( self, *scopes: str, @@ -89,12 +48,23 @@ async def get_token( enable_cae: bool = False, **kwargs: Any, ) -> AccessToken: - credential = await self.get_credential() - return credential.get_token(*scopes, claims=claims, tenant_id=tenant_id, enable_cae=enable_cae, **kwargs) + if self._credential is None: + self._credential = await self._initialize_async(self._env_client) + + if self._credential is None: + raise EvaluationException( + f"{self.__class__.__name__} could not determine the credential to use.", + target=ErrorTarget.UNKNOWN, + category=ErrorCategory.INVALID_VALUE, + blame=ErrorBlame.SYSTEM_ERROR, + ) + + return self._credential.get_token(*scopes, claims=claims, tenant_id=tenant_id, enable_cae=enable_cae, **kwargs) async def __aenter__(self) -> "AsyncAzureTokenProvider": try: - await self.get_credential() + if self._credential is None: + self._credential = await self._initialize_async(self._env_client) except BaseException: try: await self.close() diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_token_cache.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_token_cache.py deleted file mode 100644 index be74ab288085..000000000000 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_token_cache.py +++ /dev/null @@ -1,273 +0,0 @@ -# --------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# --------------------------------------------------------- - -"""Reuse a caller credential's access tokens across model requests. - -Developer credentials such as ``AzureCliCredential`` start a process for every token request, so evaluations that -requested a token for each model call were slow and, when concurrent, could fail. A token is kept per credential and -scope until shortly before it expires, as azure-core's bearer token policy does. Credentials are keyed by identity -through a weak reference and never kept alive; a credential that cannot be weakly referenced, or a token without a -numeric expiry, is not cached. A token that a service rejected is dropped, and a token whose acquisition began before -that rejection is not stored, so a rejected token never re-enters the cache through a slower request. Requests that -waited on an acquisition whose token was not cached, because of such a rejection or because of its expiry, use that -token instead of each requesting another one. - -azure-identity's ``get_bearer_token_provider`` is not used because it keeps one token per provider instance with no -coordination between concurrent requests, so concurrent first requests each call the credential; because its -wrapper never sees a service's 401 response, so a rejected token is served until it expires; and because its -asynchronous form binds its lock to one event loop while each synchronous evaluator call runs on a new loop. -Sharing one token across evaluators, waiting with a deadline and under cancellation, and dropping a rejected token -need the coordination here. -""" - -import contextlib -import inspect -import math -import threading -import time -import weakref -from typing import Any, Callable, Dict, NamedTuple, Optional, Tuple - -from azure.core.credentials_async import AsyncTokenCredential -from azure.core.exceptions import ClientAuthenticationError - -from azure.ai.evaluation._exceptions import EvaluationException - -# A token is reused until this many seconds before it expires, like azure-core's bearer token policy. -TOKEN_REFRESH_SECONDS = 300 -# How often a request waiting for another request's token checks whether it was abandoned. -TOKEN_WAIT_INTERVAL_SECONDS = 0.05 - - -def is_async_credential(credential: Any) -> bool: - """Whether a credential's token is requested on the caller's event loop rather than on a worker thread. - - That is a credential whose ``get_token`` is a coroutine function, or one that implements the asynchronous - credential protocol but not the synchronous context manager protocol: its ``get_token`` may return an awaitable - that needs the loop it was called on. Any other credential is asked on a worker thread, so a plain ``get_token`` - that needs the caller's running loop must belong to an ``AsyncTokenCredential``. - """ - return inspect.iscoroutinefunction(getattr(credential, "get_token", None)) or ( - isinstance(credential, AsyncTokenCredential) and not isinstance(credential, contextlib.AbstractContextManager) - ) - - -def _is_fresh(token: Any) -> bool: - expires_on = getattr(token, "expires_on", None) - if isinstance(expires_on, bool) or not isinstance(expires_on, (int, float)): - return False - try: - return ( - isinstance(getattr(token, "token", None), str) - and math.isfinite(expires_on) - and expires_on - time.time() > TOKEN_REFRESH_SECONDS - ) - except OverflowError: - return False - - -def _has_value(token: Any) -> bool: - """Whether a token has a string value to send. Its expiry is not checked: a credential may report none, or 0.""" - return isinstance(getattr(token, "token", None), str) - - -def describe_credential_failure(error: BaseException) -> Optional[Dict[str, Any]]: - """Describe a credential's authentication failure by its message, or return None for any other failure. - - Requests that waited on an authentication failure share it as a ``ClientAuthenticationError``. Any other failure, - such as a token service outage, is not shared, so each waiting request makes its own attempt and raises the - credential's own error type. The description never keeps the error or its traceback. - """ - if isinstance(error, ClientAuthenticationError): - return {"message": str(error) or type(error).__name__} - return None - - -class _SharedFailure(NamedTuple): - """A failed token request's description and the attempt count after it, kept for requests of the same kind.""" - - attempt: int - description: Dict[str, Any] - - -class _CredentialTokens: - """One credential's cached token for one scope, the lock for its token requests, the latest failure that each - kind of request described, and the count of rejections that invalidates acquisitions begun before them.""" - - def __init__(self, reference: Any) -> None: - self.reference = reference - self.token: Any = None - self.lock = threading.Lock() - self.attempts = 0 - # Keyed by the failure formatter and exception type of the requests that share the failure, so requests that - # describe or raise failures differently never raise each other's. Cleared when a token request succeeds. - self.failures: Dict[Tuple[Callable[..., Any], Callable[..., BaseException]], _SharedFailure] = {} - # The latest token request's token when it was not cached, because a rejection kept it out or because of its - # expiry, for the requests that waited on that request. It is never cached. - self.unstored: Any = None - # Incremented when a service rejects a token. An acquisition that began before the latest rejection may - # return the rejected token, so a token is stored only under the generation its acquisition began with. - self.generation = 0 - - -class CredentialTokenCache: - """Share each caller credential's token for a scope across requests until shortly before it expires. - - Concurrent requests that share a synchronous credential wait for one token request. A caller can also ask that - requests which waited on a failed token request share its failure instead of each requesting a token again; only - requests that describe and raise failures the same way share one. A token a service rejected is discarded; the - credential's next token is cached again, even when it is the same token, while an acquisition that began before - the rejection is never stored. Requests that waited on an acquisition whose token was not cached use that token - instead of each requesting another one. - """ - - def __init__(self) -> None: - self._entries: Dict[Tuple[int, str], _CredentialTokens] = {} - # Reentrant because a weak reference callback can run while a lookup holds the lock. - self._guard = threading.RLock() - - def _entry(self, credential: Any, scope: str, create: bool = False) -> Optional[_CredentialTokens]: - key = (id(credential), scope) - with self._guard: - entry = self._entries.get(key) - if entry is not None and entry.reference() is credential: - return entry - if not create: - return None - - def forget(reference: Any) -> None: - with self._guard: - current = self._entries.get(key) - if current is not None and current.reference is reference: - del self._entries[key] - - try: - entry = _CredentialTokens(weakref.ref(credential, forget)) - except TypeError: - return None - self._entries[key] = entry - return entry - - def cached(self, credential: Any, scope: str) -> Any: - entry = self._entry(credential, scope) - with self._guard: - token = entry.token if entry is not None else None - return token if _is_fresh(token) else None - - def generation(self, credential: Any, scope: str) -> int: - """Return the rejection generation to pass to ``store`` for a token acquisition that begins now.""" - entry = self._entry(credential, scope) - with self._guard: - return entry.generation if entry is not None else 0 - - def store(self, credential: Any, scope: str, token: Any, generation: Optional[int] = None) -> bool: - """Keep a fresh token for reuse and return whether it was kept. - - ``generation`` is what ``generation`` returned when the acquisition began. A token acquired before a later - rejection is not stored, because it may be the rejected token; without a generation the token is stored. - """ - entry = self._entry(credential, scope, create=True) if _is_fresh(token) else None - if entry is not None: - with self._guard: - if generation is None or generation == entry.generation: - entry.token = token - return True - return False - - def discard(self, credential: Any, scope: str, token: str) -> None: - """Stop reusing a token a service rejected, and invalidate acquisitions that began before the rejection.""" - entry = self._entry(credential, scope, create=True) - if entry is not None: - with self._guard: - entry.generation += 1 - if entry.token is not None and entry.token.token == token: - entry.token = None - if entry.unstored is not None and entry.unstored.token == token: - entry.unstored = None - - def fetch( - self, - credential: Any, - scope: str, - *, - abandoned: Optional[threading.Event] = None, - deadline: Optional[float] = None, - describe_failure: Optional[Callable[[BaseException], Optional[Dict[str, Any]]]] = None, - failure_type: Callable[..., BaseException] = EvaluationException, - ) -> Any: - """Request a synchronous credential's token once for concurrent requests. - - Returns None, without requesting a token, when the request was abandoned or its monotonic ``deadline`` passed - while it waited for another request's token. With ``describe_failure``, a request that waited on a failed - request of its own kind, one that passed the same ``describe_failure`` and ``failure_type``, raises - ``failure_type`` built from that request's description instead of requesting a token again. A None - description is not shared, and a request never raises a failure that another kind of request described, so - each keeps its own error type and classification; with no failure of its own kind, it makes its own attempt. - The description must not keep the error, whose traceback references the credential, and ``describe_failure`` - and ``failure_type`` must be module-level functions or classes: they key the shared failures, so a bound method - or closure would keep what it references, such as the credential, alive. Requests that waited on a request - whose token was not cached use that token. - """ - entry = self._entry(credential, scope, create=True) - if entry is None: - if (abandoned is not None and abandoned.is_set()) or ( - deadline is not None and time.monotonic() >= deadline - ): - return None - return credential.get_token(scope) - with self._guard: - queued_after = entry.attempts - # Wait in short steps so an abandoned request stops waiting promptly. - while True: - remaining = math.inf if deadline is None else deadline - time.monotonic() - if (abandoned is not None and abandoned.is_set()) or remaining <= 0: - return None - if entry.lock.acquire(timeout=min(remaining, TOKEN_WAIT_INTERVAL_SECONDS)): - break - try: - token = self.cached(credential, scope) - if ( - token is not None - or (abandoned is not None and abandoned.is_set()) - or (deadline is not None and time.monotonic() >= deadline) - ): - return token - kind = (describe_failure, failure_type) - with self._guard: - waited = entry.attempts != queued_after - failure = entry.failures.get(kind) if waited and describe_failure is not None else None - unstored = entry.unstored if waited else None - if failure is not None and failure.attempt > queued_after: - raise failure_type(**failure.description) - if _has_value(unstored): - # Issued for the request this one waited on and not rejected since it was shared, so use it rather - # than request another token. - return unstored - with self._guard: - generation = entry.generation - try: - token = credential.get_token(scope) - except Exception as error: - description = describe_failure(error) if describe_failure is not None else None - with self._guard: - entry.attempts += 1 - if description is not None: - entry.failures[kind] = _SharedFailure(entry.attempts, description) - else: - entry.failures.pop(kind, None) - entry.unstored = None - raise - # Decide whether the token is cached and publish the outcome together, so a rejection cannot land between - # them and be undone. - with self._guard: - stored = inspect.isawaitable(token) or self.store(credential, scope, token, generation) - entry.attempts += 1 - entry.failures.clear() - entry.unstored = token if not stored and _has_value(token) else None - return token - finally: - entry.lock.release() - - -TOKENS = CredentialTokenCache() diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py index fa1ad1b89b1e..87876d997321 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py @@ -3,19 +3,17 @@ # --------------------------------------------------------- import asyncio -import inspect import re -import threading from logging import Logger from os import PathLike from pathlib import Path -from typing import Any, AsyncGenerator, Dict, Final, List, Mapping, Optional, Sequence, Tuple, Union, cast +from typing import Any, AsyncGenerator, Awaitable, Dict, Final, List, Mapping, Optional, Sequence, Tuple, Union, cast -from openai import AsyncAzureOpenAI, AsyncOpenAI, AuthenticationError, NotGiven, OpenAIError +from openai import AsyncAzureOpenAI, AsyncOpenAI, NotGiven, OpenAIError +from openai.lib.azure import AsyncAzureADTokenProvider from azure.core.credentials import TokenCredential from azure.core.credentials_async import AsyncTokenCredential -from azure.core.exceptions import ClientAuthenticationError from azure.ai.evaluation._exceptions import ErrorTarget from azure.ai.evaluation._constants import DefaultOpenEncoding, TokenScope @@ -43,7 +41,6 @@ from azure.ai.evaluation._constants import DEFAULT_MAX_COMPLETION_TOKENS_REASONING_MODELS from azure.ai.evaluation._legacy._common._logging import get_logger from azure.ai.evaluation._legacy._common._async_token_provider import AsyncAzureTokenProvider -from azure.ai.evaluation._legacy._common._token_cache import TOKENS, describe_credential_failure, is_async_credential from azure.ai.evaluation._user_agent import UserAgentSingleton PROMPTY_EXTENSION: Final[str] = ".prompty" @@ -299,7 +296,6 @@ async def __call__( # pylint: disable=docstring-keyword-should-match-keyword-on default_headers = {"User-Agent": UserAgentSingleton().value} api_client: Union[AsyncAzureOpenAI, AsyncOpenAI, AsyncByoProjectResponsesClient] - token_provider: Optional[_ReusedTokenProvider] = None if is_byo: # BYO judge model (connection/deployment): route chat.completions through the Foundry # Responses API, which resolves the connection and auth (API key / managed identity / OAuth2). @@ -315,15 +311,15 @@ async def __call__( # pylint: disable=docstring-keyword-should-match-keyword-on else: connection = Connection.parse_from_config(configuration) if isinstance(connection, AzureOpenAIConnection): - if not connection.api_key: - token_provider = self.get_token_provider(self._token_credential) api_client = AsyncAzureOpenAI( azure_endpoint=connection.azure_endpoint, api_key=connection.api_key, azure_deployment=connection.azure_deployment, api_version=connection.api_version, max_retries=max_retries, - azure_ad_token_provider=token_provider, + azure_ad_token_provider=( + self.get_token_provider(self._token_credential) if not connection.api_key else None + ), default_headers=default_headers, ) elif isinstance(connection, OpenAIConnection): @@ -339,17 +335,11 @@ async def __call__( # pylint: disable=docstring-keyword-should-match-keyword-on f"'{type(connection).__name__}' is not a supported connection type.", target=ErrorTarget.EVAL_RUN ) - try: - response: OpenAIChatResponseType = await self._send_with_retries( - api_client=api_client, - params=params, - timeout=timeout, - ) - except WrappedOpenAIError as error: - if token_provider is not None and isinstance(error.__cause__, AuthenticationError): - # A rejected token is not reused. - token_provider.discard() - raise + response: OpenAIChatResponseType = await self._send_with_retries( + api_client=api_client, + params=params, + timeout=timeout, + ) return await format_llm_response( response=response, @@ -441,75 +431,18 @@ async def _send_with_retries( retry += 1 @staticmethod - def get_token_provider(cred: Union[TokenCredential, AsyncTokenCredential]) -> "_ReusedTokenProvider": + def get_token_provider(cred: Union[TokenCredential, AsyncTokenCredential]) -> AsyncAzureADTokenProvider: """Get the token provider for the prompty. :param Union[TokenCredential, AsyncTokenCredential] cred: The Azure authentication credential. - :return: A token provider for the chat client that reuses the credential's token. - :rtype: _ReusedTokenProvider + :return: The token provider if a credential is provided, otherwise None. + :rtype: Optional[AsyncAzureADTokenProvider] """ - return _ReusedTokenProvider(cred, TokenScope.COGNITIVE_SERVICES_MANAGEMENT) + async def _wrapper() -> str: + token = cred.get_token(TokenScope.COGNITIVE_SERVICES_MANAGEMENT) + if isinstance(token, Awaitable): + token = await token + return token.token -class _ReusedTokenProvider: - """Supply a credential's token to a chat client, reusing it until shortly before it expires. - - Concurrent requests that share a synchronous credential wait for one token request, on a worker thread so the - event loop is not blocked. An asynchronous credential, one whose ``get_token`` is a coroutine function or that - implements ``AsyncTokenCredential`` but not the synchronous context manager protocol, is asked on the caller's - event loop instead; a plain ``get_token`` that needs the running loop must therefore belong to an - ``AsyncTokenCredential``. The default token provider is represented by the synchronous credential it creates. The - request that made a failed token request raises the credential's error. When that failure was an authentication - failure, requests that waited on it raise a ``ClientAuthenticationError`` with the same message instead of each - requesting a token again; after any other failure, each waiting request makes its own attempt. A token the service - rejected is discarded, and an acquisition that began before the rejection is not stored. - """ - - def __init__(self, credential: Union[TokenCredential, AsyncTokenCredential], scope: str) -> None: - self._credential = credential - self._scope = scope - self._source: Any = credential - self._token: Optional[str] = None - - async def __call__(self) -> str: - credential: Any = self._credential - if isinstance(credential, AsyncAzureTokenProvider): - # Its tokens come from a synchronous credential, so concurrent requests can share one token request. - credential = await credential.get_credential() - self._source = credential - token = TOKENS.cached(credential, self._scope) - if token is None: - # An acquisition that begins before a service rejects a token may return that token; the generation it - # began under lets the cache refuse it. - generation = TOKENS.generation(credential, self._scope) - if is_async_credential(credential): - # An asynchronous credential's token request may need this event loop. - token = credential.get_token(self._scope) - if inspect.isawaitable(token): - token = await token - TOKENS.store(credential, self._scope, token, generation) - else: - abandoned = threading.Event() - try: - # A request waiting for another request's token holds a default-executor thread while it waits; - # past the executor's size, further requests queue for a thread before they reach the cache. - token = await asyncio.to_thread( - TOKENS.fetch, - credential, - self._scope, - abandoned=abandoned, - describe_failure=describe_credential_failure, - failure_type=ClientAuthenticationError, - ) - finally: - abandoned.set() - if inspect.isawaitable(token): - token = await token - TOKENS.store(credential, self._scope, token, generation) - self._token = token.token - return self._token - - def discard(self) -> None: - """Stop reusing the token this provider last supplied, which the service rejected.""" - if self._token is not None: - TOKENS.discard(self._source, self._scope, self._token) + return _wrapper diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index fbd5c3484eab..eeaad0c7bac4 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -5,16 +5,12 @@ """Offline tests for shared decision-model runners, base selection, and evaluation strategy.""" import asyncio -import gc import hashlib import json import logging import math -import pickle import sys import threading -import time -import weakref from copy import deepcopy from logging.handlers import BufferingHandler from concurrent.futures import ThreadPoolExecutor @@ -23,13 +19,11 @@ import pytest from openai.types.chat import ChatCompletion -from azure.core.credentials import AccessToken from azure.core.pipeline.policies import AsyncRedirectPolicy, AsyncRetryPolicy from azure.core.pipeline.transport import AsyncHttpTransport, HttpResponse -from azure.core.exceptions import ClientAuthenticationError, ServiceRequestError +from azure.core.exceptions import ServiceRequestError from azure.ai.evaluation import CoherenceEvaluator, FluencyEvaluator -from azure.ai.evaluation._constants import TokenScope from azure.ai.evaluation._evaluators._common import EvaluatorBase, ModelEvaluatorBase, PromptyEvaluatorBase from azure.ai.evaluation._evaluators._common import _model_runners from azure.ai.evaluation._evaluators._common._decision_strategy import ( @@ -144,42 +138,6 @@ def run(runner, **overrides): return asyncio.run(runner(**arguments)) -# Only bounds a hung test; no assertion depends on how long anything takes. -WAIT_SECONDS = 10 - - -def watch_token_queue(credential, requests): - """Return an event set once ``requests`` threads have queued for the credential's Decision token request. - - Each request reads the cache's attempt count before it queues, so a token request released after this event - completes while every request is queued, whatever the thread scheduling. - """ - entry = _model_runners._TOKENS._entry(credential, _model_runners._DECISION_TOKEN_SCOPE, create=True) - lock, queued, all_queued = entry.lock, set(), threading.Event() - - class QueueWatchingLock: - def acquire(self, *args, **kwargs): - queued.add(threading.get_ident()) - if len(queued) >= requests: - all_queued.set() - return lock.acquire(*args, **kwargs) - - def release(self): - lock.release() - - entry.lock = QueueWatchingLock() - return all_queued - - -class PicklableCredential: - def __init__(self): - self.requested = 0 - - def get_token(self, *_, **__): - self.requested += 1 - return AccessToken("picklable", int(time.time()) + 3600) - - class TestDecisionModelRunnerConfiguration: @pytest.mark.parametrize( "changes", @@ -691,563 +649,6 @@ def test_unsupported_question_type(self, client): assert not client.calls -class TestDecisionTokenReuse: - """A credential's token serves repeated and concurrent requests until shortly before it expires.""" - - CONFIG = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} - - @staticmethod - def access_token(value, lifetime=3600): - return AccessToken(value, int(time.time()) + lifetime) - - @staticmethod - def authorizations(client): - return [call["headers"]["Authorization"] for call in client.calls] - - @pytest.mark.parametrize("asynchronous", [False, True]) - def test_fresh_token_is_reused_across_requests_and_runners(self, client, asynchronous): - client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - token = self.access_token("shared-token") - credential = Mock() - credential.get_token = AsyncMock(return_value=token) if asynchronous else Mock(return_value=token) - first, second = DecisionModelRunner(self.CONFIG, credential), DecisionModelRunner(self.CONFIG, credential) - for runner in (first, second, first): - run(runner) - credential.get_token.assert_called_once_with("https://cognitiveservices.azure.com/.default") - assert self.authorizations(client) == ["Bearer shared-token"] * 3 - - @pytest.mark.parametrize("lifetime", [None, 60], ids=["no-expiry", "near-expiry"]) - def test_tokens_without_a_usable_expiry_are_requested_each_time(self, client, lifetime): - client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - token = SimpleNamespace(token="per-request") if lifetime is None else self.access_token("per-request", lifetime) - credential = Mock(get_token=Mock(return_value=token)) - runner = DecisionModelRunner(self.CONFIG, credential) - run(runner) - run(runner) - assert credential.get_token.call_count == 2 - - def test_concurrent_requests_share_one_synchronous_token_request(self, client): - client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - release = threading.Event() - requested = [] - - def get_token(*scopes, **_): - requested.append(scopes) - release.wait() - return self.access_token("single-request") - - credential = Mock(get_token=Mock(side_effect=get_token)) - all_queued = watch_token_queue(credential, 6) - runners = [DecisionModelRunner(self.CONFIG, credential) for _ in range(6)] - # Like evaluate(), each runner awaits on its own thread and event loop. - with ThreadPoolExecutor(max_workers=6) as pool: - try: - requests = [pool.submit(run, runner, timeout=2 * WAIT_SECONDS) for runner in runners] - assert all_queued.wait(WAIT_SECONDS) - finally: - release.set() - for request in requests: - request.result(2 * WAIT_SECONDS) - assert len(requested) == 1 - assert self.authorizations(client) == ["Bearer single-request"] * 6 - - @pytest.mark.parametrize("expiry", ["none", "near", "past", "zero"]) - def test_concurrent_requests_share_a_token_that_is_not_cached(self, client, expiry): - client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - release = threading.Event() - requested = [] - - def get_token(*_, **__): - requested.append(1) - release.wait() - expires_on = {"none": None, "near": time.time() + 120, "past": time.time() - 1, "zero": 0}[expiry] - return SimpleNamespace(token="uncached", expires_on=expires_on) - - credential = Mock(get_token=Mock(side_effect=get_token)) - all_queued = watch_token_queue(credential, 6) - with ThreadPoolExecutor(max_workers=6) as pool: - try: - requests = [ - pool.submit(run, DecisionModelRunner(self.CONFIG, credential), timeout=2 * WAIT_SECONDS) - for _ in range(6) - ] - assert all_queued.wait(WAIT_SECONDS) - finally: - release.set() - for request in requests: - request.result(2 * WAIT_SECONDS) - assert len(requested) == 1 - assert self.authorizations(client) == ["Bearer uncached"] * 6 - assert _model_runners._TOKENS.cached(credential, _model_runners._DECISION_TOKEN_SCOPE) is None - - def test_decision_tokens_are_kept_apart_from_chat_tokens(self, client): - client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - credential = Mock(get_token=Mock(return_value=self.access_token("decision"))) - run(DecisionModelRunner(self.CONFIG, credential)) - credential.get_token.assert_called_once_with("https://cognitiveservices.azure.com/.default") - # Chat requests key the cache with their own scope, so they never reuse a Decision token. - assert _model_runners._TOKENS.cached(credential, TokenScope.COGNITIVE_SERVICES_MANAGEMENT) is None - - def test_rejected_token_is_not_reused(self, client): - client.response = FakeResponse(status_code=401, body=b"{}") - tokens = iter([self.access_token("rejected"), self.access_token("renewed")]) - credential = Mock(get_token=Mock(side_effect=lambda *_, **__: next(tokens))) - runner = DecisionModelRunner(self.CONFIG, credential) - with pytest.raises(EvaluationException, match="HTTP 401"): - run(runner) - client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - run(runner) - run(runner) - assert credential.get_token.call_count == 2 - assert self.authorizations(client) == ["Bearer rejected", "Bearer renewed", "Bearer renewed"] - - @pytest.mark.parametrize("status", [403, 429, 500]) - def test_a_token_is_kept_after_other_errors(self, client, status): - client.response = FakeResponse(status_code=status, body=b"{}") - credential = Mock(get_token=Mock(return_value=self.access_token("kept"))) - runner = DecisionModelRunner(self.CONFIG, credential, max_retries=0) - with pytest.raises(EvaluationException, match=f"HTTP {status}"): - run(runner) - client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - run(runner) - credential.get_token.assert_called_once() - assert self.authorizations(client) == ["Bearer kept"] * 2 - - def test_a_token_the_credential_reissues_after_a_rejection_is_cached_again(self, client): - client.response = FakeResponse(status_code=401, body=b"{}") - credential = Mock(get_token=Mock(return_value=self.access_token("same"))) - runner = DecisionModelRunner(self.CONFIG, credential) - with pytest.raises(EvaluationException, match="HTTP 401"): - run(runner) - assert _model_runners._TOKENS.cached(credential, _model_runners._DECISION_TOKEN_SCOPE) is None - # The service accepts the token now, for example after a role assignment propagated. The credential - # re-issues the same token: it is requested once more, then reused again instead of once per request. - client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - for _ in range(3): - run(runner) - assert credential.get_token.call_count == 2 - assert self.authorizations(client) == ["Bearer same"] * 4 - assert _model_runners._TOKENS.cached(credential, _model_runners._DECISION_TOKEN_SCOPE).token == "same" - - def test_an_acquisition_that_began_before_two_rejections_is_not_stored(self): - scope = _model_runners._DECISION_TOKEN_SCOPE - - async def scenario(): - entered, gate = asyncio.Event(), asyncio.Event() - calls = [] - - class AsyncCredential: - async def get_token(self, *_, **__): - calls.append(1) - call = len(calls) - if call == 1: - entered.set() - await gate.wait() - # The delayed first acquisition and the second return token A; later ones return token B. - return TestDecisionTokenReuse.access_token("token-A" if call <= 2 else "token-B") - - credential = AsyncCredential() - runner = DecisionModelRunner(self.CONFIG, credential) - deadline = time.monotonic() + WAIT_SECONDS - pending = asyncio.ensure_future(runner._get_token(deadline)) - await entered.wait() - # Two later acquisitions complete first, and the service rejects each token with a 401. - for expected in ("token-A", "token-B"): - assert await runner._get_token(deadline) == expected - _model_runners._TOKENS.discard(credential, scope, expected) - gate.set() - # The delayed acquisition returns a token the service already rejected; it must not be cached. - assert await pending == "token-A" - assert _model_runners._TOKENS.cached(credential, scope) is None - # The next acquisition is cached again. - assert await runner._get_token(deadline) == "token-B" - assert _model_runners._TOKENS.cached(credential, scope).token == "token-B" - - asyncio.run(scenario()) - - def test_credential_without_weak_references_is_used_without_caching(self, client): - client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - requested = [] - credential = SimpleNamespace(get_token=lambda *scopes, **_: requested.append(scopes) or self.access_token("t")) - runner = DecisionModelRunner(self.CONFIG, credential) - run(runner) - run(runner) - assert len(requested) == 2 - - def test_unhashable_credential_is_cached_by_identity(self, client): - from dataclasses import dataclass - - @dataclass - class Credential: - requested: int = 0 - - def get_token(self, *_, **__): - self.requested += 1 - return TestDecisionTokenReuse.access_token("by-identity") - - client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - credential = Credential() - runner = DecisionModelRunner(self.CONFIG, credential) - run(runner) - run(runner) - assert credential.requested == 1 - - def test_equal_credentials_keep_their_own_tokens(self, client): - class Credential: - def __init__(self, token): - self.token = token - - def __eq__(self, other): - return isinstance(other, Credential) - - def __hash__(self): - return 0 - - def get_token(self, *_, **__): - return TestDecisionTokenReuse.access_token(self.token) - - client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - first, second = Credential("first-identity"), Credential("second-identity") - assert first == second - run(DecisionModelRunner(self.CONFIG, first)) - run(DecisionModelRunner(self.CONFIG, second)) - assert self.authorizations(client) == ["Bearer first-identity", "Bearer second-identity"] - - def test_request_whose_deadline_passes_while_waiting_does_not_request_a_token(self, client): - client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - entered, release = threading.Event(), threading.Event() - requested = [] - - def get_token(*_, **__): - requested.append(1) - entered.set() - release.wait() - raise RuntimeError("Azure CLI is not logged in") - - credential = Mock(get_token=Mock(side_effect=get_token)) - outcomes = {} - - def request(name, timeout): - try: - run(DecisionModelRunner(self.CONFIG, credential), timeout=timeout) - except EvaluationException as error: - outcomes[name] = str(error) - - leader = threading.Thread(target=request, args=("leader", 2 * WAIT_SECONDS)) - waiter = threading.Thread(target=request, args=("waiter", 0.3)) - try: - leader.start() - assert entered.wait(WAIT_SECONDS) - # The first request holds the credential's token request until the second request's deadline has passed. - waiter.start() - waiter.join(WAIT_SECONDS) - waiter_finished = not waiter.is_alive() - finally: - release.set() - leader.join(WAIT_SECONDS) - assert waiter_finished and not leader.is_alive() - assert len(requested) == 1 - assert "timed out" in outcomes["waiter"] - assert "Could not acquire a token" in outcomes["leader"] - - @pytest.mark.parametrize( - "failure,token_requests,blame,message", - [ - pytest.param( - lambda: ClientAuthenticationError("Please run 'az login' (private-detail)"), - 1, - ErrorBlame.USER_ERROR, - "Could not acquire a token", - id="authentication-shared-at-once", - ), - pytest.param( - lambda: EvaluationException( - message="Service-owned Decision credential acquisition failed (RuntimeError, status 5xx).", - blame=ErrorBlame.SYSTEM_ERROR, - category=ErrorCategory.FAILED_EXECUTION, - target=ErrorTarget.EVALUATE, - ), - 4, - ErrorBlame.SYSTEM_ERROR, - "acquisition failed", - id="other-not-shared-each-request-tries", - ), - ], - ) - def test_requests_waiting_on_a_failed_token_request_share_its_failure( - self, client, failure, token_requests, blame, message - ): - client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - entered, release = threading.Event(), threading.Event() - requested = [] - - def get_token(*_, **__): - requested.append(1) - entered.set() - release.wait() - raise failure() - - credential = Mock(get_token=Mock(side_effect=get_token)) - all_queued = watch_token_queue(credential, 4) - errors = {} - - def request(name): - try: - run(DecisionModelRunner(self.CONFIG, credential), timeout=2 * WAIT_SECONDS) - except EvaluationException as error: - errors[name] = error - - leader = threading.Thread(target=request, args=("leader",)) - waiters = [threading.Thread(target=request, args=(f"waiter-{index}",)) for index in range(3)] - try: - leader.start() - assert entered.wait(WAIT_SECONDS) - for waiter in waiters: - waiter.start() - assert all_queued.wait(WAIT_SECONDS) - finally: - release.set() - for thread in [leader, *waiters]: - thread.join(WAIT_SECONDS) - # Queued requests share an authentication failure instead of each starting another request; any other failure is - # not shared, so each request makes its own attempt and keeps its own typed error. - assert len(requested) == token_requests and len(errors) == 4 - assert all(error.blame == blame and message in str(error) for error in errors.values()) - assert all("private-detail" not in str(error) for error in errors.values()) - credential.get_token.side_effect = lambda *_, **__: self.access_token("recovered") - run(DecisionModelRunner(self.CONFIG, credential)) - assert credential.get_token.call_count == token_requests + 1 and self.authorizations(client) == [ - "Bearer recovered" - ] - - def test_requests_waiting_on_a_transient_token_failure_recover_with_one_more_request(self, client): - client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - entered, release = threading.Event(), threading.Event() - requested = [] - - def get_token(*_, **__): - requested.append(1) - if len(requested) == 1: - entered.set() - release.wait() - raise RuntimeError("token service unavailable") - return self.access_token("recovered") - - credential = Mock(get_token=Mock(side_effect=get_token)) - all_queued = watch_token_queue(credential, 4) - outcomes = {} - - def request(name): - try: - run(DecisionModelRunner(self.CONFIG, credential), timeout=2 * WAIT_SECONDS) - outcomes[name] = "completed" - except EvaluationException as error: - outcomes[name] = str(error) - - leader = threading.Thread(target=request, args=("leader",)) - waiters = [threading.Thread(target=request, args=(f"waiter-{index}",)) for index in range(3)] - try: - leader.start() - assert entered.wait(WAIT_SECONDS) - for waiter in waiters: - waiter.start() - assert all_queued.wait(WAIT_SECONDS) - finally: - release.set() - for thread in [leader, *waiters]: - thread.join(WAIT_SECONDS) - # Only the request whose token request failed fails; the next request recovers the token for the others. - assert len(requested) == 2 - assert "Could not acquire a token" in outcomes["leader"] - assert [outcomes[f"waiter-{index}"] for index in range(3)] == ["completed"] * 3 - assert self.authorizations(client) == ["Bearer recovered"] * 3 - - def test_requests_waiting_on_a_token_kept_out_of_the_cache_use_it(self, client): - client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - entered, release = threading.Event(), threading.Event() - requested = [] - - def get_token(*_, **__): - requested.append(1) - entered.set() - release.wait() - return self.access_token("issued") - - credential = Mock(get_token=Mock(side_effect=get_token)) - all_queued = watch_token_queue(credential, 4) - scope = _model_runners._DECISION_TOKEN_SCOPE - with ThreadPoolExecutor(max_workers=4) as pool: - try: - requests = [ - pool.submit(run, DecisionModelRunner(self.CONFIG, credential), timeout=2 * WAIT_SECONDS) - for _ in range(4) - ] - assert entered.wait(WAIT_SECONDS) and all_queued.wait(WAIT_SECONDS) - # Another request's 401 lands while the token is requested, so the token is not cached. - _model_runners._TOKENS.discard(credential, scope, "rejected-earlier") - finally: - release.set() - for request in requests: - request.result(2 * WAIT_SECONDS) - assert len(requested) == 1 - assert self.authorizations(client) == ["Bearer issued"] * 4 - assert _model_runners._TOKENS.cached(credential, scope) is None - # A request that starts afterwards requests its own token, which is cached again. - run(DecisionModelRunner(self.CONFIG, credential)) - assert len(requested) == 2 - assert _model_runners._TOKENS.cached(credential, scope).token == "issued" - - def test_a_token_rejected_before_its_request_resumes_is_not_stored_again(self, client, monkeypatch): - client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - credential = Mock(get_token=Mock(return_value=self.access_token("shared"))) - fetch = _model_runners._TOKENS.fetch - scope = _model_runners._DECISION_TOKEN_SCOPE - - def fetch_then_reject(*args, **kwargs): - token = fetch(*args, **kwargs) - # Another request's 401 discards the token before this request's event loop resumes. - _model_runners._TOKENS.discard(credential, scope, token.token) - return token - - monkeypatch.setattr(_model_runners._TOKENS, "fetch", fetch_then_reject) - run(DecisionModelRunner(self.CONFIG, credential)) - assert _model_runners._TOKENS.cached(credential, scope) is None - - def test_failed_token_request_does_not_keep_the_credential_alive(self, client): - class Credential: - def get_token(self, *_, **__): - raise RuntimeError("Azure CLI is not logged in") - - credential = Credential() - with pytest.raises(EvaluationException, match="Could not acquire a token"): - run(DecisionModelRunner(self.CONFIG, credential)) - reference = weakref.ref(credential) - del credential - gc.collect() - assert reference() is None - - def test_cancelled_request_stops_waiting_for_another_token_request(self, client): - client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - entered, release = threading.Event(), threading.Event() - requested = [] - - def get_token(*_, **__): - requested.append(1) - entered.set() - release.wait() - return self.access_token("late") - - credential = Mock(get_token=Mock(side_effect=get_token)) - all_queued = watch_token_queue(credential, 2) - cancelled = threading.Event() - outcomes = {} - - def waiter(): - try: - run( - DecisionModelRunner(self.CONFIG, credential, cancellation_requested=cancelled.is_set), - timeout=2 * WAIT_SECONDS, - ) - except EvaluationException as error: - outcomes["waiter"] = str(error) - - leader = threading.Thread( - target=lambda: run(DecisionModelRunner(self.CONFIG, credential), timeout=2 * WAIT_SECONDS) - ) - waiting = threading.Thread(target=waiter) - try: - leader.start() - assert entered.wait(WAIT_SECONDS) - waiting.start() - # Cancel only once the second request waits for the first request's token. - assert all_queued.wait(WAIT_SECONDS) - cancelled.set() - # The first request's token request stays blocked until ``finally``, so the cancelled request can only - # return by giving up its wait. - waiting.join(WAIT_SECONDS) - still_waiting = waiting.is_alive() - finally: - release.set() - leader.join(WAIT_SECONDS) - waiting.join(WAIT_SECONDS) - assert not still_waiting and "cancelled" in outcomes["waiter"] - assert len(requested) == 1 - - def test_cache_does_not_keep_the_credential_alive(self, client): - client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - token = self.access_token("caller-owned") - - class Credential: - def get_token(self, *_, **__): - return token - - credential = Credential() - run(DecisionModelRunner(self.CONFIG, credential)) - assert _model_runners._TOKENS.cached(credential, _model_runners._DECISION_TOKEN_SCOPE) is token - reference = weakref.ref(credential) - del credential - gc.collect() - assert reference() is None - - def test_a_runner_still_pickles_after_its_token_is_cached(self, client): - client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - credential = PicklableCredential() - runner = DecisionModelRunner(self.CONFIG, credential) - run(runner) - copy = pickle.loads(pickle.dumps(runner)) - # The copy's credential is another object, so it requests its own token once and then reuses it. - run(copy) - run(copy) - run(runner) - assert credential.requested == 1 - assert copy._credential.requested == 2 - assert self.authorizations(client) == ["Bearer picklable"] * 4 - - @pytest.mark.parametrize("asynchronous", [False, True]) - @pytest.mark.parametrize( - "token", - [None, SimpleNamespace(token=None, expires_on=None), SimpleNamespace(token=123, expires_on=None)], - ids=["none", "no-token", "non-string-token"], - ) - def test_a_credential_that_returns_no_usable_token_is_a_user_error(self, client, token, asynchronous): - credential = Mock() - credential.get_token = AsyncMock(return_value=token) if asynchronous else Mock(return_value=token) - with pytest.raises(EvaluationException, match="Could not acquire a token") as raised: - run(DecisionModelRunner(self.CONFIG, credential)) - assert raised.value.blame == ErrorBlame.USER_ERROR - assert not client.calls - - def test_a_token_wait_ends_at_the_deadline_without_requesting_a_token(self): - credential = Mock(get_token=Mock(return_value=self.access_token("unused"))) - scope = _model_runners._DECISION_TOKEN_SCOPE - entry = _model_runners._TOKENS._entry(credential, scope, create=True) - runner = DecisionModelRunner(self.CONFIG, credential) - # Another request holds the credential's token request throughout. - assert entry.lock.acquire(timeout=WAIT_SECONDS) - try: - assert _model_runners._TOKENS.fetch(credential, scope, deadline=time.monotonic() + 0.1) is None - assert asyncio.run(runner._get_token(time.monotonic() + 0.1)) is None - finally: - entry.lock.release() - credential.get_token.assert_not_called() - - def test_a_credential_without_weak_references_is_not_asked_after_the_deadline(self): - requested = [] - credential = SimpleNamespace(get_token=lambda *scopes, **_: requested.append(scopes)) - scope = _model_runners._DECISION_TOKEN_SCOPE - - assert _model_runners._TOKENS.fetch(credential, scope, deadline=time.monotonic() - 1) is None - assert requested == [] - - def test_a_request_whose_token_wait_reached_the_deadline_times_out(self, client, monkeypatch): - runner = DecisionModelRunner(self.CONFIG, Mock(get_token=Mock())) - monkeypatch.setattr(runner, "_get_token", AsyncMock(return_value=None)) - with pytest.raises(EvaluationException, match="timed out") as raised: - run(runner) - assert raised.value.blame == ErrorBlame.SYSTEM_ERROR - assert not client.calls - - class TestAnswerValidation: def test_noul(self): assert validate_noul_answer({"type": "noul", "noul": 0.25}) == 0.25 diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_token_reuse.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_token_reuse.py deleted file mode 100644 index ad568952ce2f..000000000000 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_token_reuse.py +++ /dev/null @@ -1,891 +0,0 @@ -# --------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# --------------------------------------------------------- - -"""The chat flow's token provider reuses a caller credential's token, kept apart from Decision tokens.""" - -import asyncio -import gc -import threading -import time -import weakref -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import MagicMock - -import httpx -import pytest -from azure.core.credentials import AccessToken -from azure.core.exceptions import ClientAuthenticationError -from openai import AuthenticationError, InternalServerError - -import azure.ai.evaluation._evaluators._coherence as coherence -from azure.ai.evaluation._evaluators._common import _model_runners -from azure.ai.evaluation._common.utils import construct_prompty_model_config, validate_model_config -from azure.ai.evaluation._constants import TokenScope -from azure.ai.evaluation._exceptions import EvaluationException -from azure.ai.evaluation._legacy._common._async_token_provider import AsyncAzureTokenProvider -from azure.ai.evaluation._legacy._common._token_cache import TOKENS, describe_credential_failure, is_async_credential -from azure.ai.evaluation._legacy.prompty import AsyncPrompty -from azure.ai.evaluation._legacy.prompty._exceptions import WrappedOpenAIError - -CHAT_SCOPE = TokenScope.COGNITIVE_SERVICES_MANAGEMENT -DECISION_SCOPE = _model_runners._DECISION_TOKEN_SCOPE -COHERENCE_PROMPTY = Path(coherence.__file__).parent / "coherence.prompty" -CHAT_MODEL = construct_prompty_model_config( - validate_model_config({"azure_endpoint": "https://example.openai.azure.com", "azure_deployment": "judge"}), - "2024-10-21", - "test-agent", -) -# Only bounds a hung test; no assertion depends on how long anything takes. -WAIT_SECONDS = 10 - - -def access_token(value): - return AccessToken(value, int(time.time()) + 3600) - - -def request_token(credential): - return asyncio.run(AsyncPrompty.get_token_provider(credential)()) - - -def join_all(threads): - for thread in threads: - thread.join(WAIT_SECONDS) - assert not any(thread.is_alive() for thread in threads) - - -def watch_token_queue(credential, requests): - """Return an event set once ``requests`` threads have queued for the credential's chat token request. - - Each request reads the cache's attempt count before it queues, so a token request released after this event - completes while every request is queued, whatever the thread scheduling. - """ - entry = TOKENS._entry(credential, CHAT_SCOPE, create=True) - lock, queued, all_queued = entry.lock, set(), threading.Event() - - class QueueWatchingLock: - def acquire(self, *args, **kwargs): - queued.add(threading.get_ident()) - if len(queued) >= requests: - all_queued.set() - return lock.acquire(*args, **kwargs) - - def release(self): - lock.release() - - entry.lock = QueueWatchingLock() - return all_queued - - -class Credential: - """A synchronous credential that records each token request and answers from a queue of results. - - With ``gate``, each token request waits until the test sets the event; tests set it in ``finally``. - """ - - def __init__(self, *results, gate=None): - self.scopes = [] - self._results = list(results) - self._gate = gate - self._lock = threading.Lock() - - def get_token(self, *scopes, **_): - with self._lock: - self.scopes.append(scopes) - result = self._results.pop(0) if len(self._results) > 1 else self._results[0] - if self._gate is not None: - self._gate.wait() - if isinstance(result, Exception): - raise result - return result - - -@pytest.mark.unittest -class TestChatTokenReuse: - def test_a_fresh_token_is_reused_across_requests_and_flows(self): - credential = Credential(access_token("chat")) - - assert [request_token(credential) for _ in range(3)] == ["chat"] * 3 - assert asyncio.run(AsyncPrompty.get_token_provider(credential)()) == "chat" - assert credential.scopes == [(CHAT_SCOPE,)] - - def test_concurrent_requests_wait_for_one_token_request(self): - release = threading.Event() - credential = Credential(access_token("shared"), gate=release) - all_queued = watch_token_queue(credential, 8) - tokens = [] - threads = [threading.Thread(target=lambda: tokens.append(request_token(credential))) for _ in range(8)] - try: - for thread in threads: - thread.start() - assert all_queued.wait(WAIT_SECONDS) - finally: - release.set() - join_all(threads) - - assert tokens == ["shared"] * 8 - assert credential.scopes == [(CHAT_SCOPE,)] - - @pytest.mark.parametrize("expiry", ["none", "near", "past", "zero"]) - def test_concurrent_requests_share_a_token_that_is_not_cached(self, expiry): - expires_on = {"none": None, "near": time.time() + 120, "past": time.time() - 1, "zero": 0}[expiry] - issued = SimpleNamespace(token="uncached", expires_on=expires_on) - release = threading.Event() - credential = Credential(issued, gate=release) - all_queued = watch_token_queue(credential, 6) - tokens = [] - threads = [threading.Thread(target=lambda: tokens.append(request_token(credential))) for _ in range(6)] - try: - for thread in threads: - thread.start() - assert all_queued.wait(WAIT_SECONDS) - finally: - release.set() - join_all(threads) - - assert tokens == ["uncached"] * 6 - assert len(credential.scopes) == 1 - assert TOKENS.cached(credential, CHAT_SCOPE) is None - # A request that did not wait on that token request makes its own. - assert request_token(credential) == "uncached" - assert len(credential.scopes) == 2 - - def test_a_token_without_a_value_is_not_given_to_waiting_requests(self): - release = threading.Event() - credential = Credential(SimpleNamespace(token=None, expires_on=None), gate=release) - all_queued = watch_token_queue(credential, 3) - outcomes = [] - threads = [threading.Thread(target=lambda: outcomes.append(request_token(credential))) for _ in range(3)] - try: - for thread in threads: - thread.start() - assert all_queued.wait(WAIT_SECONDS) - finally: - release.set() - join_all(threads) - - # Each request asks the credential itself and returns its value, as before. - assert outcomes == [None] * 3 - assert len(credential.scopes) == 3 - - def test_waiting_for_another_requests_token_does_not_block_the_event_loop(self): - entered, release = threading.Event(), threading.Event() - - class SlowCredential(Credential): - def get_token(self, *scopes, **kwargs): - entered.set() - return super().get_token(*scopes, **kwargs) - - credential = SlowCredential(access_token("shared"), gate=release) - all_queued = watch_token_queue(credential, 2) - leader = threading.Thread(target=lambda: request_token(credential)) - # Releases the token request if waiting blocked the loop, so a regression fails instead of hanging. - watchdog = threading.Timer(WAIT_SECONDS, release.set) - - async def scenario(): - waiting = asyncio.ensure_future(AsyncPrompty.get_token_provider(credential)()) - assert await asyncio.to_thread(all_queued.wait, WAIT_SECONDS) - # The loop ran this coroutine while the request waits for the first request's token. - assert not waiting.done() and not release.is_set() - release.set() - return await waiting - - try: - watchdog.start() - leader.start() - assert entered.wait(WAIT_SECONDS) - assert asyncio.run(scenario()) == "shared" - finally: - release.set() - watchdog.cancel() - join_all([leader]) - assert len(credential.scopes) == 1 - - def test_chat_and_decision_tokens_are_kept_apart(self): - credential = Credential(access_token("chat")) - TOKENS.store(credential, DECISION_SCOPE, access_token("decision")) - - assert request_token(credential) == "chat" - assert credential.scopes == [(CHAT_SCOPE,)] - assert TOKENS.cached(credential, DECISION_SCOPE).token == "decision" - - def test_a_failed_token_request_raises_the_credential_error_and_is_retried(self): - credential = Credential(ClientAuthenticationError("Please run az login."), access_token("recovered")) - - with pytest.raises(ClientAuthenticationError, match="az login"): - request_token(credential) - assert request_token(credential) == "recovered" - assert len(credential.scopes) == 2 - - def test_requests_queued_behind_a_failed_token_request_share_its_failure(self): - release = threading.Event() - credential = Credential( - ClientAuthenticationError("Please run az login."), access_token("recovered"), gate=release - ) - all_queued = watch_token_queue(credential, 4) - outcomes = [] - - def request(): - try: - outcomes.append(request_token(credential)) - except ClientAuthenticationError as error: - outcomes.append(str(error)) - - threads = [threading.Thread(target=request) for _ in range(4)] - try: - for thread in threads: - thread.start() - assert all_queued.wait(WAIT_SECONDS) - finally: - release.set() - join_all(threads) - - assert outcomes == ["Please run az login."] * 4 - assert len(credential.scopes) == 1 - # A request made after the failure tries again. - assert request_token(credential) == "recovered" - - def test_other_failures_are_not_shared_so_each_request_keeps_the_error_type(self): - class TokenServiceOutage(Exception): - pass - - release = threading.Event() - credential = Credential(TokenServiceOutage("The token service is unavailable."), gate=release) - all_queued = watch_token_queue(credential, 3) - outcomes = [] - - def request(): - try: - outcomes.append(request_token(credential)) - except Exception as error: # pylint: disable=broad-except - outcomes.append(type(error).__name__) - - threads = [threading.Thread(target=request) for _ in range(3)] - try: - for thread in threads: - thread.start() - assert all_queued.wait(WAIT_SECONDS) - finally: - release.set() - join_all(threads) - - assert outcomes == ["TokenServiceOutage"] * 3 - assert len(credential.scopes) == 3 - - def test_requests_waiting_on_a_token_kept_out_of_the_cache_use_it(self): - entered, release = threading.Event(), threading.Event() - - class SlowCredential(Credential): - def get_token(self, *scopes, **kwargs): - entered.set() - release.wait() - return super().get_token(*scopes, **kwargs) - - credential = SlowCredential(access_token("issued")) - all_queued = watch_token_queue(credential, 4) - tokens = [] - threads = [threading.Thread(target=lambda: tokens.append(request_token(credential))) for _ in range(4)] - try: - for thread in threads: - thread.start() - assert entered.wait(WAIT_SECONDS) and all_queued.wait(WAIT_SECONDS) - # Another request's 401 lands while the token is requested, so the token is not cached. - TOKENS.discard(credential, CHAT_SCOPE, "rejected-earlier") - finally: - release.set() - join_all(threads) - - assert tokens == ["issued"] * 4 - assert len(credential.scopes) == 1 - assert TOKENS.cached(credential, CHAT_SCOPE) is None - # A request that starts afterwards requests its own token, which is cached again. - assert request_token(credential) == "issued" - assert len(credential.scopes) == 2 - assert TOKENS.cached(credential, CHAT_SCOPE).token == "issued" - - def test_a_rejected_token_is_not_given_to_waiting_requests(self): - credential = Credential(access_token("unused")) - entry = TOKENS._entry(credential, CHAT_SCOPE, create=True) - entry.unstored = access_token("issued") - - TOKENS.discard(credential, CHAT_SCOPE, "other") - assert entry.unstored.token == "issued" - TOKENS.discard(credential, CHAT_SCOPE, "issued") - assert entry.unstored is None - - def test_a_rejection_cannot_land_between_caching_and_the_hand_off_to_waiting_requests(self, monkeypatch): - credential = Credential(access_token("issued")) - store = TOKENS.store - guard_free = [] - - def probe_guard(): - # A rejection from another request needs this lock. - acquired = TOKENS._guard.acquire(blocking=False) - if acquired: - TOKENS._guard.release() - guard_free.append(acquired) - - def store_then_probe(*args, **kwargs): - # A rejection while the token was requested keeps it out of the cache for waiting requests. - TOKENS.discard(credential, CHAT_SCOPE, "rejected-earlier") - stored = store(*args, **kwargs) - probe = threading.Thread(target=probe_guard) - probe.start() - join_all([probe]) - return stored - - monkeypatch.setattr(TOKENS, "store", store_then_probe) - assert request_token(credential) == "issued" - - # The decision to cache and the hand-off happen while the lock is held, so no rejection lands between them. - assert guard_free == [False] - assert TOKENS._entry(credential, CHAT_SCOPE).unstored.token == "issued" - # A rejection afterwards still reaches the handed-off token. - TOKENS.discard(credential, CHAT_SCOPE, "issued") - assert TOKENS._entry(credential, CHAT_SCOPE).unstored is None - - def test_requests_waiting_on_a_failed_token_request_do_not_get_an_older_token(self): - class TokenServiceOutage(Exception): - pass - - entered, first_release, second_release = threading.Event(), threading.Event(), threading.Event() - calls = [] - - class StagedCredential(Credential): - def get_token(self, *scopes, **kwargs): - calls.append(1) - if len(calls) == 1: - entered.set() - first_release.wait() - elif len(calls) == 2: - second_release.wait() - return super().get_token(*scopes, **kwargs) - - credential = StagedCredential( - access_token("issued"), TokenServiceOutage("The token service is unavailable."), access_token("recovered") - ) - results = {} - - def request(name): - try: - results[name] = request_token(credential) - except Exception as error: # pylint: disable=broad-except - results[name] = type(error).__name__ - - # A rejection while the first token is requested keeps that token out of the cache. - first = threading.Thread(target=request, args=("first",)) - try: - first.start() - assert entered.wait(WAIT_SECONDS) - TOKENS.discard(credential, CHAT_SCOPE, "rejected-earlier") - finally: - first_release.set() - join_all([first]) - assert results["first"] == "issued" and TOKENS.cached(credential, CHAT_SCOPE) is None - - # The next token request fails; the requests waiting on it make their own attempt. - all_queued = watch_token_queue(credential, 3) - names = ["second", "waiter-1", "waiter-2"] - threads = [threading.Thread(target=request, args=(name,)) for name in names] - try: - for thread in threads: - thread.start() - assert all_queued.wait(WAIT_SECONDS) - finally: - second_release.set() - join_all(threads) - - assert sorted(results[name] for name in names) == ["TokenServiceOutage", "recovered", "recovered"] - assert len(calls) == 3 - - def test_a_token_without_an_expiry_is_requested_each_time(self): - credential = Credential(SimpleNamespace(token="no-expiry", expires_on=None)) - - assert [request_token(credential) for _ in range(2)] == ["no-expiry"] * 2 - assert len(credential.scopes) == 2 - - def test_an_asynchronous_credentials_token_is_reused(self): - class AsyncCredential: - def __init__(self): - self.calls = 0 - - async def get_token(self, *_, **__): - self.calls += 1 - return access_token("async") - - credential = AsyncCredential() - - assert [request_token(credential) for _ in range(3)] == ["async"] * 3 - assert credential.calls == 1 - - def test_the_cache_does_not_keep_the_credential_alive(self): - credential = Credential(access_token("caller-owned")) - assert request_token(credential) == "caller-owned" - assert TOKENS.cached(credential, CHAT_SCOPE) is not None - - reference = weakref.ref(credential) - del credential - gc.collect() - assert reference() is None - - @pytest.mark.parametrize("default_provider", [False, True], ids=["caller credential", "default provider"]) - @pytest.mark.parametrize( - "error_type, status, tokens_sent", - [(AuthenticationError, 401, ["rejected", "replacement"]), (InternalServerError, 500, ["rejected", "rejected"])], - ) - def test_only_a_token_rejected_with_401_is_requested_again( - self, monkeypatch, error_type, status, tokens_sent, default_provider - ): - credential = Credential(access_token("rejected"), access_token("replacement")) - if default_provider: - provider = AsyncAzureTokenProvider() - provider._credential = credential - token_credential = provider - else: - token_credential = credential - flow = AsyncPrompty.load(source=COHERENCE_PROMPTY, model=CHAT_MODEL, token_credential=token_credential) - sent = [] - - async def send(self, *, api_client, params, timeout): - sent.append(await api_client._azure_ad_token_provider()) - response = httpx.Response(status, request=httpx.Request("POST", "https://example.openai.azure.com")) - error = error_type("request failed", response=response, body=None) - raise WrappedOpenAIError(error=error) from error - - monkeypatch.setattr(AsyncPrompty, "_send_with_retries", send) - for _ in range(2): - with pytest.raises(WrappedOpenAIError): - asyncio.run(flow(query="When does the museum open?", response="At noon.")) - - assert sent == tokens_sent - assert len(credential.scopes) == len(set(tokens_sent)) - - def test_the_default_provider_reuses_its_token(self): - provider = AsyncAzureTokenProvider() - inner = Credential(access_token("default")) - provider._credential = inner - - assert [request_token(provider) for _ in range(3)] == ["default"] * 3 - assert inner.scopes == [(CHAT_SCOPE,)] - asyncio.run(provider.close()) - - def test_concurrent_requests_share_the_default_providers_token_request(self): - release = threading.Event() - provider = AsyncAzureTokenProvider() - inner = Credential(access_token("default"), gate=release) - provider._credential = inner - all_queued = watch_token_queue(inner, 6) - tokens = [] - threads = [threading.Thread(target=lambda: tokens.append(request_token(provider))) for _ in range(6)] - try: - for thread in threads: - thread.start() - assert all_queued.wait(WAIT_SECONDS) - finally: - release.set() - join_all(threads) - - assert tokens == ["default"] * 6 - assert inner.scopes == [(CHAT_SCOPE,)] - asyncio.run(provider.close()) - - def test_the_default_provider_creates_its_credential_once(self, monkeypatch): - inner = Credential(access_token("created")) - created = [] - - async def initialize(client): - created.append(client) - return inner - - monkeypatch.setattr(AsyncAzureTokenProvider, "_initialize_async", staticmethod(initialize)) - provider = AsyncAzureTokenProvider() - - assert [request_token(provider) for _ in range(3)] == ["created"] * 3 - assert len(created) == 1 and inner.scopes == [(CHAT_SCOPE,)] - # Its own get_token still asks the credential it created. - assert asyncio.run(provider.get_token(CHAT_SCOPE)).token == "created" - assert len(inner.scopes) == 2 - asyncio.run(provider.close()) - - def test_concurrent_first_requests_share_one_default_credential_and_token_request(self, monkeypatch): - requests = 4 - barrier = threading.Barrier(requests) - created, closed = [], [] - - class ClosableCredential(Credential): - def close(self): - closed.append(self) - - async def initialize(client): - # Every first request creates a credential at the same time. - await asyncio.to_thread(barrier.wait, WAIT_SECONDS) - credential = ClosableCredential(access_token("default")) - created.append(credential) - return credential - - monkeypatch.setattr(AsyncAzureTokenProvider, "_initialize_async", staticmethod(initialize)) - provider = AsyncAzureTokenProvider() - tokens = [] - threads = [threading.Thread(target=lambda: tokens.append(request_token(provider))) for _ in range(requests)] - for thread in threads: - thread.start() - join_all(threads) - - assert tokens == ["default"] * requests - # The first credential created is kept and the others are closed, so one token request serves every row. - assert len(created) == requests and len(closed) == requests - 1 - assert sum(len(credential.scopes) for credential in created) == 1 - asyncio.run(provider.close()) - - def test_a_credential_created_while_the_default_provider_closes_is_closed(self, monkeypatch): - closed = [] - - class ClosableCredential(Credential): - def close(self): - closed.append(self) - - created = ClosableCredential(access_token("late")) - - async def scenario(): - entered, gate = asyncio.Event(), asyncio.Event() - - async def initialize(client): - entered.set() - await gate.wait() - return created - - monkeypatch.setattr(AsyncAzureTokenProvider, "_initialize_async", staticmethod(initialize)) - provider = AsyncAzureTokenProvider() - pending = asyncio.ensure_future(provider.get_credential()) - await entered.wait() - await provider.close() - gate.set() - with pytest.raises(EvaluationException, match="already been closed"): - await pending - assert provider._credential is None - - asyncio.run(scenario()) - assert closed == [created] - - def test_a_token_rejected_while_another_request_awaits_it_is_not_stored_again(self): - async def scenario(): - entered, gate = asyncio.Event(), asyncio.Event() - calls = [] - - class AsyncCredential: - async def get_token(self, *_, **__): - calls.append(1) - if len(calls) == 1: - entered.set() - await gate.wait() - return access_token("rejected") - - credential = AsyncCredential() - waiting = AsyncPrompty.get_token_provider(credential) - rejected = AsyncPrompty.get_token_provider(credential) - pending = asyncio.ensure_future(waiting()) - # The first request misses the cache and awaits its token. - await entered.wait() - assert await rejected() == "rejected" - # The service rejects that token with a 401 before the first request resumes. - rejected.discard() - gate.set() - assert await pending == "rejected" - assert TOKENS.cached(credential, CHAT_SCOPE) is None - # A different token from the credential is cached again. - TOKENS.store(credential, CHAT_SCOPE, access_token("new")) - assert TOKENS.cached(credential, CHAT_SCOPE).token == "new" - - asyncio.run(scenario()) - - def test_a_token_the_credential_reissues_after_a_rejection_is_cached_again(self): - credential = Credential(access_token("same")) - - async def reject_once(): - provider = AsyncPrompty.get_token_provider(credential) - assert await provider() == "same" - # The service rejects the token with a 401, for example while a role assignment still propagates. - provider.discard() - - asyncio.run(reject_once()) - assert TOKENS.cached(credential, CHAT_SCOPE) is None - # The credential re-issues the same token: it is requested once more, then reused again. - assert [request_token(credential) for _ in range(3)] == ["same"] * 3 - assert len(credential.scopes) == 2 - assert TOKENS.cached(credential, CHAT_SCOPE).token == "same" - - def test_an_acquisition_that_began_before_two_rejections_is_not_stored(self): - async def scenario(): - entered, gate = asyncio.Event(), asyncio.Event() - calls = [] - - class AsyncCredential: - async def get_token(self, *_, **__): - calls.append(1) - call = len(calls) - if call == 1: - entered.set() - await gate.wait() - # The delayed first acquisition and the second return token A; later ones return token B. - return access_token("token-A" if call <= 2 else "token-B") - - credential = AsyncCredential() - pending = asyncio.ensure_future(AsyncPrompty.get_token_provider(credential)()) - await entered.wait() - # Two later acquisitions complete first, and the service rejects each token with a 401. - for expected in ("token-A", "token-B"): - provider = AsyncPrompty.get_token_provider(credential) - assert await provider() == expected - provider.discard() - gate.set() - # The delayed acquisition returns a token the service already rejected; it must not be cached. - assert await pending == "token-A" - assert TOKENS.cached(credential, CHAT_SCOPE) is None - # The next acquisition is cached again. - assert await AsyncPrompty.get_token_provider(credential)() == "token-B" - assert TOKENS.cached(credential, CHAT_SCOPE).token == "token-B" - - asyncio.run(scenario()) - - -DECISION_CONFIG = { - "type": "decision", - "azure_endpoint": "https://example.services.ai.azure.com", - "azure_deployment": "Decision-1", -} - - -def request_decision_token(credential): - runner = _model_runners.DecisionModelRunner(DECISION_CONFIG, credential) - return asyncio.run(runner._get_token(time.monotonic() + 2 * WAIT_SECONDS)) - - -class GatedCredential(Credential): - """A ``Credential`` whose token requests wait until the test sets ``release``, and that reports the first one.""" - - def __init__(self, *results): - self.entered, self.release = threading.Event(), threading.Event() - super().__init__(*results, gate=self.release) - - def get_token(self, *scopes, **kwargs): - self.entered.set() - return super().get_token(*scopes, **kwargs) - - -def queue_behind(credential, leader, *waiters): - """Return the outcomes of ``leader``'s token request and of each of ``waiters``, all queued behind it.""" - requests = [leader, *waiters] - all_queued = watch_token_queue(credential, len(requests)) - outcomes = [None] * len(requests) - - def request(index): - try: - outcomes[index] = requests[index](credential) - except Exception as error: # pylint: disable=broad-except - outcomes[index] = error - - threads = [threading.Thread(target=request, args=(index,)) for index in range(len(requests))] - try: - threads[0].start() - assert credential.entered.wait(WAIT_SECONDS) - for thread in threads[1:]: - thread.start() - assert all_queued.wait(WAIT_SECONDS) - finally: - credential.release.set() - join_all(threads) - return outcomes - - -def fetch_as(describe_failure, failure_type): - """Return a request for the credential's chat-scope token that describes and raises failures its own way.""" - - def request(credential): - return TOKENS.fetch(credential, CHAT_SCOPE, describe_failure=describe_failure, failure_type=failure_type).token - - return request - - -# Two kinds of request that describe and raise failures differently, as chat and Decision requests do. -AUTHENTICATION_KIND = fetch_as(describe_credential_failure, ClientAuthenticationError) -EVALUATION_KIND = fetch_as(_model_runners._token_failure, EvaluationException) -EVALUATION_MESSAGE = "Could not acquire a token for the decision model." - - -@pytest.mark.unittest -class TestFailuresAreSharedWithinOneKindOfRequest: - """Requests that describe failures differently share a credential's tokens, but not each other's failures.""" - - @pytest.mark.parametrize("leader", ["authentication", "evaluation"]) - def test_both_kinds_of_request_share_one_token_request(self, leader): - kinds = {"authentication": AUTHENTICATION_KIND, "evaluation": EVALUATION_KIND} - waiter = "evaluation" if leader == "authentication" else "authentication" - credential = GatedCredential(access_token("shared")) - - assert queue_behind(credential, kinds[leader], kinds[waiter]) == ["shared", "shared"] - assert len(credential.scopes) == 1 - - @pytest.mark.parametrize("retried", ["recovered", "failed again"]) - def test_an_evaluation_failure_is_not_replayed_to_another_kind_of_request(self, retried): - retry = access_token("recovered") if retried == "recovered" else ClientAuthenticationError("Run az login.") - credential = GatedCredential(ClientAuthenticationError("Please run az login."), retry) - - evaluation, authentication = queue_behind(credential, EVALUATION_KIND, AUTHENTICATION_KIND) - - assert isinstance(evaluation, ClientAuthenticationError) - # The other request asks the credential itself rather than raising the evaluation description as its own - # authentication error. - if retried == "recovered": - assert authentication == "recovered" - else: - assert isinstance(authentication, ClientAuthenticationError) and str(authentication) == "Run az login." - assert len(credential.scopes) == 2 - - @pytest.mark.parametrize("retried", ["recovered", "failed again"]) - def test_an_authentication_failure_is_not_replayed_to_another_kind_of_request(self, retried): - retry = access_token("recovered") if retried == "recovered" else ClientAuthenticationError("Run az login.") - credential = GatedCredential(ClientAuthenticationError("Please run az login."), retry) - - authentication, evaluation = queue_behind(credential, AUTHENTICATION_KIND, EVALUATION_KIND) - - assert isinstance(authentication, ClientAuthenticationError) and "az login" in str(authentication) - # The other request asks the credential itself rather than raising an unclassified EvaluationException. - if retried == "recovered": - assert evaluation == "recovered" - else: - assert isinstance(evaluation, ClientAuthenticationError) and str(evaluation) == "Run az login." - assert len(credential.scopes) == 2 - - def test_evaluation_failures_are_shared_whichever_request_wakes_first(self): - credential = GatedCredential(ClientAuthenticationError("Please run az login.")) - - evaluation, authentication, other_evaluation = queue_behind( - credential, EVALUATION_KIND, AUTHENTICATION_KIND, EVALUATION_KIND - ) - - assert isinstance(evaluation, ClientAuthenticationError) - # The other kind of request makes its own attempt and raises the credential's error. The request of the same - # kind shares the first failure even when the other kind's attempt ends first. - assert isinstance(authentication, ClientAuthenticationError) and str(authentication) == "Please run az login." - assert isinstance(other_evaluation, EvaluationException) and EVALUATION_MESSAGE in str(other_evaluation) - assert len(credential.scopes) == 2 - - def test_authentication_failures_are_shared_whichever_request_wakes_first(self): - credential = GatedCredential(ClientAuthenticationError("Please run az login.")) - - authentication, evaluation, other_authentication = queue_behind( - credential, AUTHENTICATION_KIND, EVALUATION_KIND, AUTHENTICATION_KIND - ) - - assert isinstance(authentication, ClientAuthenticationError) - assert isinstance(evaluation, ClientAuthenticationError) and not isinstance(evaluation, EvaluationException) - assert isinstance(other_authentication, ClientAuthenticationError) - assert str(other_authentication) == "Please run az login." - assert len(credential.scopes) == 2 - - def test_a_successful_token_request_ends_sharing_of_earlier_failures(self): - credential = Credential(ClientAuthenticationError("Please run az login."), access_token("recovered")) - - with pytest.raises(ClientAuthenticationError): - EVALUATION_KIND(credential) - entry = TOKENS._entry(credential, CHAT_SCOPE) - assert list(entry.failures) == [(_model_runners._token_failure, EvaluationException)] - - assert AUTHENTICATION_KIND(credential) == "recovered" - # A request that waited on the failure but wakes after this success gets the token, not the earlier failure. - assert entry.failures == {} - - -class LoopBoundAsyncCredential: - """An asynchronous credential whose ``get_token`` is a plain function that needs the caller's event loop.""" - - def __init__(self): - self.calls = 0 - - def get_token(self, *_, **__): - self.calls += 1 - loop = asyncio.get_running_loop() - - async def request(): - assert asyncio.get_running_loop() is loop - return access_token("loop-bound") - - return request() - - async def close(self): - pass - - async def __aenter__(self): - return self - - async def __aexit__(self, *_): - pass - - -class DualProtocolCredential(GatedCredential): - """A synchronous credential that implements both the synchronous and the asynchronous context manager protocols.""" - - def close(self): - pass - - def __enter__(self): - return self - - def __exit__(self, *_): - pass - - async def __aenter__(self): - return self - - async def __aexit__(self, *_): - pass - - -@pytest.mark.unittest -class TestAsynchronousCredentialRequests: - @pytest.mark.parametrize("kind", ["chat", "decision"]) - def test_an_asynchronous_credential_requests_its_token_on_the_callers_event_loop(self, kind): - make_request = {"chat": request_token, "decision": request_decision_token}[kind] - credential = LoopBoundAsyncCredential() - - assert [make_request(credential) for _ in range(2)] == ["loop-bound"] * 2 - assert credential.calls == 1 - - def test_a_synchronous_credential_with_both_context_manager_protocols_is_asked_on_a_worker_thread(self): - credential = DualProtocolCredential(access_token("dual")) - - # Both requests wait for one token request on a worker thread; on the event loop each would ask the credential. - assert queue_behind(credential, request_token, request_token) == ["dual", "dual"] - assert len(credential.scopes) == 1 - - def test_a_plain_get_token_outside_the_asynchronous_protocol_is_asked_on_a_worker_thread(self): - class MinimalWrapper: - def get_token(self, *_, **__): - return asyncio.ensure_future(asyncio.sleep(0, access_token("minimal"))) - - # A plain get_token that needs the running event loop must belong to an AsyncTokenCredential. - assert not is_async_credential(MinimalWrapper()) - - def test_synchronous_identity_credentials_request_tokens_on_a_worker_thread(self): - from azure.identity import ( - AzureCliCredential, - ClientSecretCredential, - DefaultAzureCredential, - ManagedIdentityCredential, - ) - from azure.identity.aio import AzureCliCredential as AsyncAzureCliCredential - - synchronous = [ - AzureCliCredential(), - ClientSecretCredential("tenant", "client", "secret"), - DefaultAzureCredential(exclude_shared_token_cache_credential=True), - ManagedIdentityCredential(), - Credential(access_token("sync")), - MagicMock(get_token=MagicMock(return_value=access_token("mock")), close=MagicMock()), - DualProtocolCredential(access_token("dual")), - ] - - assert not any(is_async_credential(credential) for credential in synchronous) - assert is_async_credential(AsyncAzureCliCredential()) - assert is_async_credential(LoopBoundAsyncCredential()) From 9e48c7954fe627723c39fb11a48dc15261a3a00f Mon Sep 17 00:00:00 2001 From: Ahmad Qardahji Date: Tue, 3 Nov 2026 15:45:51 -0800 Subject: [PATCH 09/14] Treat empty tool call fields as absent in Tool Call Success A tool_calls, function_call or tool_call_id field that is None or an empty string, list or dict no longer sends a text-only response to the judge. OpenAI's model_dump() writes tool_calls: None and function_call: None on every reply, so those replies reached the chat or Decision judge and could pass. Populated fields and values of any other type still go to the judge. --- .../azure-ai-evaluation/CHANGELOG.md | 2 +- .../_tool_call_success/_tool_call_success.py | 16 +++-- .../test_tool_call_success_evaluator.py | 64 ++++++++++++++++++- 3 files changed, 76 insertions(+), 6 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index 07af8fbbc849..1f1e0af6e38e 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -13,7 +13,7 @@ ### Bugs Fixed - `_ToolCallSuccessEvaluator` no longer scores successful tool calls 0 when a chat judge model, such as gpt-4.1 or gpt-4o, puts `score` and `status` inside `properties`. The example outputs in its prompt are now valid JSON, and `score`, `status` and `reason` are read from `properties` when the output has none at the top level. A missing score is no longer reported as 0, and any score other than 0 or 1 raises a system-classified `EvaluationException`, as `_ToolInputAccuracyEvaluator` does. -- `_ToolCallSuccessEvaluator` reports a list of messages that holds no tool calls, such as one that contains only text or only MCP approval messages, as not applicable without calling the chat or Decision judge model. Strings, text that contains `[TOOL_CALL]` or `[TOOL_RESULT]`, and messages with any other content are still sent to the judge model. +- `_ToolCallSuccessEvaluator` reports a list of messages that holds no tool calls, such as one that contains only text or only MCP approval messages, as not applicable without calling the chat or Decision judge model. A `tool_calls`, `function_call` or `tool_call_id` field that is None or empty, as in OpenAI's `model_dump()` of a reply without tool calls, does not count as a tool call. Strings, text that contains `[TOOL_CALL]` or `[TOOL_RESULT]`, and messages with any other content are still sent to the judge model. ## 1.18.10 (2026-10-22) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py index 127b5c9052ad..5a43040055ff 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py @@ -150,7 +150,8 @@ def __call__( result = evaluator(response=response, ) A list of messages that holds only text has no tool calls, so it is reported as not applicable without - calling the judge model. + calling the judge model. A tool_calls, function_call or tool_call_id field that is None or empty does not + count as a tool call. :keyword response: The response being evaluated, either a string or a list of messages (full agent response potentially including tool calls) @@ -311,15 +312,22 @@ def _binary_score(value: Any) -> Optional[float]: return None +def _holds_nothing(value: Any) -> bool: + """Return True for None and an empty string, list or dict.""" + return value is None or (isinstance(value, (str, list, dict)) and not value) + + def _has_no_tool_calls(messages) -> bool: """Return True only for a list of text messages and MCP approvals that holds no tool calls, results or traces. - A list in any other format may hold tool activity this evaluator does not parse, so the judge assesses it. + A tool_calls, function_call or tool_call_id field that holds nothing, such as the None that OpenAI's model_dump() + writes, counts as absent. A list in any other format may hold tool activity this evaluator does not parse, so the + judge assesses it. """ if not isinstance(messages, list) or not messages: return False for message in messages: - if not isinstance(message, dict) or any(key in message for key in _TOOL_CALL_KEYS): + if not isinstance(message, dict) or not all(_holds_nothing(message.get(key)) for key in _TOOL_CALL_KEYS): return False content = message.get("content") items = content @@ -329,7 +337,7 @@ def _has_no_tool_calls(messages) -> bool: # Only MCP approval requests or responses, which are not tool calls; a response names its request # in tool_call_id. continue - if message.get("role") not in _CONVERSATION_ROLES or "tool_call_id" in message: + if message.get("role") not in _CONVERSATION_ROLES or not _holds_nothing(message.get("tool_call_id")): return False if isinstance(content, str): texts = [content] diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py index 5cb144c81ee3..9380a53c9fdd 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py @@ -47,6 +47,16 @@ {"role": "user", "content": [{"type": "text", "text": "What is the weather in Paris?"}]}, {"role": "assistant", "content": [{"type": "output_text", "text": "I can't check the weather."}]}, ] +# What OpenAI's ChatCompletionMessage.model_dump() returns for a reply without tool calls. +OPENAI_TEXT_REPLY = { + "content": "Hello.", + "refusal": None, + "role": "assistant", + "annotations": [], + "audio": None, + "function_call": None, + "tool_calls": None, +} JUDGE_USAGE = {"input_token_count": 11, "output_token_count": 7, "total_token_count": 18} @@ -240,8 +250,23 @@ class TestToolCallSuccessWithoutToolCalls: "content": [{"type": "mcp_approval_response", "approve": False}], }, ], + [{"role": "assistant", "content": "Hello.", "tool_calls": None}], + [{"role": "assistant", "content": [{"type": "text", "text": "Hello."}], "tool_calls": []}], + [{"role": "assistant", "content": "Hello.", "function_call": None}], + [{"role": "user", "content": "Hi.", "tool_call_id": None}, {"role": "assistant", "content": "Hello."}], + [OPENAI_TEXT_REPLY], + ], + ids=[ + "text-items", + "string-content", + "every-text-role", + "mcp-approval-only", + "null-tool-calls", + "empty-tool-calls", + "null-function-call", + "null-tool-call-id", + "openai-model-dump", ], - ids=["text-items", "string-content", "every-text-role", "mcp-approval-only"], ) def test_a_response_without_tool_calls_is_not_applicable_without_a_judge_call(self, config, response): evaluator = _ToolCallSuccessEvaluator(model_config=dict(config)) @@ -305,6 +330,25 @@ def test_a_response_without_tool_calls_is_not_applicable_without_a_judge_call(se }, {"role": "assistant", "content": [{"type": "text", "text": "Done."}]}, ], + [ + { + **OPENAI_TEXT_REPLY, + "content": "Checking the weather.", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "fetch_weather", "arguments": '{"city": "Paris"}'}, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [{"type": "tool_result", "tool_result": {"error": "timed out"}}], + }, + {**OPENAI_TEXT_REPLY, "content": "The weather service timed out."}, + ], ], ids=[ "tool-call", @@ -314,6 +358,7 @@ def test_a_response_without_tool_calls_is_not_applicable_without_a_judge_call(se "trace-in-message", "trace-in-text-item", "approval-with-tool-call", + "openai-model-dump-tool-call", ], ) def test_tool_calls_and_text_responses_still_go_to_the_judge(self, response): @@ -341,6 +386,17 @@ def test_a_decision_judge_still_gets_tool_calls(self): [{"role": "assistant", "content": [{"type": "tool_call", "name": "t", "arguments": {}}]}], [{"role": "assistant", "content": "Calling a tool.", "tool_calls": [{"id": "call_1"}]}], [{"role": "assistant", "content": "Calling a tool.", "function_call": {"name": "t"}}], + [ + { + "role": "assistant", + "content": "Calling a tool.", + "tool_calls": [{"id": "call_1"}], + "function_call": None, + } + ], + [{"role": "assistant", "content": "Calling a tool.", "tool_calls": [None]}], + [{"role": "assistant", "content": "Calling a tool.", "function_call": "t"}], + [{"role": "assistant", "content": "Calling a tool.", "tool_calls": False}], [{"role": "assistant", "content": "Result.", "tool_call_id": "call_1"}], [{"role": "function", "content": "Result."}], [{"content": "No role."}], @@ -361,6 +417,11 @@ def test_text_messages_have_no_tool_calls(self): assert _has_no_tool_calls(TEXT_RESPONSE) is True assert _has_no_tool_calls([{"role": "assistant", "content": []}]) is True + @pytest.mark.parametrize("key", ["tool_calls", "function_call", "tool_call_id"]) + @pytest.mark.parametrize("value", [None, "", [], {}], ids=["none", "empty-string", "empty-list", "empty-dict"]) + def test_a_call_field_that_holds_nothing_counts_as_absent(self, key, value): + assert _has_no_tool_calls([{"role": "assistant", "content": "Hello.", key: value}]) is True + def test_mcp_approvals_are_not_tool_calls_but_a_tool_call_beside_one_is(self): approval = {"type": "mcp_approval_request", "name": "search", "arguments": "{}"} response = {"type": "mcp_approval_response", "approve": True} @@ -385,3 +446,4 @@ def test_mcp_approvals_are_not_tool_calls_but_a_tool_call_beside_one_is(self): ] ) assert not _has_no_tool_calls([{"role": "assistant", "content": [approval], "function_call": {"name": "t"}}]) + assert _has_no_tool_calls([{"role": "assistant", "content": [approval], "tool_calls": None}]) From 4fe63162768abcf1760cec12353e2f4ab28eabf6 Mon Sep 17 00:00:00 2001 From: Ahmad Qardahji Date: Tue, 3 Nov 2026 15:50:52 -0800 Subject: [PATCH 10/14] Count only passed and failed rows in binary_aggregate evaluate() divided an evaluator's passes by every row, so not applicable and errored rows counted as failures and binary_aggregate disagreed with the mean score. It now divides by the rows that passed or failed, and an evaluator without such rows gets no binary_aggregate, as one without scores gets no mean. --- .../azure-ai-evaluation/CHANGELOG.md | 1 + .../ai/evaluation/_evaluate/_evaluate.py | 12 ++++++---- .../tests/unittests/test_evaluate.py | 18 ++++++++++++++ .../test_tool_call_success_evaluator.py | 24 +++++++++++++++++++ 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index 1f1e0af6e38e..4a1bf120a6a8 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -14,6 +14,7 @@ - `_ToolCallSuccessEvaluator` no longer scores successful tool calls 0 when a chat judge model, such as gpt-4.1 or gpt-4o, puts `score` and `status` inside `properties`. The example outputs in its prompt are now valid JSON, and `score`, `status` and `reason` are read from `properties` when the output has none at the top level. A missing score is no longer reported as 0, and any score other than 0 or 1 raises a system-classified `EvaluationException`, as `_ToolInputAccuracyEvaluator` does. - `_ToolCallSuccessEvaluator` reports a list of messages that holds no tool calls, such as one that contains only text or only MCP approval messages, as not applicable without calling the chat or Decision judge model. A `tool_calls`, `function_call` or `tool_call_id` field that is None or empty, as in OpenAI's `model_dump()` of a reply without tool calls, does not count as a tool call. Strings, text that contains `[TOOL_CALL]` or `[TOOL_RESULT]`, and messages with any other content are still sent to the judge model. +- `evaluate()` computes `.binary_aggregate` over the rows that passed or failed. Rows without either result, such as not applicable or errored rows, counted as failures, so the metric disagreed with the mean score. An evaluator without any passed or failed rows has no `binary_aggregate`, as one without scores has no mean. ## 1.18.10 (2026-10-22) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py index d432380653e9..c8590defbb8e 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py @@ -273,7 +273,9 @@ def _aggregation_binary_output(df: pd.DataFrame) -> Dict[str, float]: """ Aggregate binary output results (pass/fail) from evaluation dataframe. - For each evaluator, calculates the proportion of "pass" results. + For each evaluator, calculates the proportion of "pass" results among the rows that passed or failed. Rows + without either result, such as not applicable or errored rows, are left out, as they are from the mean score, + and an evaluator without any passed or failed rows gets no binary aggregate. :param df: The dataframe of evaluation results. :type df: ~pandas.DataFrame @@ -312,14 +314,14 @@ def _aggregation_binary_output(df: pd.DataFrame) -> Dict[str, float]: ) continue - # Calculate the proportion of EVALUATION_PASS_FAIL_MAPPING[True] results - total_rows = len(df) pass_count = value_counts.get(EVALUATION_PASS_FAIL_MAPPING[True], 0) - proportion = pass_count / total_rows if total_rows > 0 else 0.0 + judged_rows = pass_count + value_counts.get(EVALUATION_PASS_FAIL_MAPPING[False], 0) + if judged_rows == 0: + continue # Set the result with the evaluator name as the key result_key = f"{evaluator_name}.{BINARY_AGGREGATE_SUFFIX}" - results[result_key] = round(proportion, 2) + results[result_key] = round(pass_count / judged_rows, 2) return results diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index 79ed493819aa..848484ab6d47 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -782,6 +782,24 @@ def test_binary_aggregation_skips_unhashable_result_columns(self, caplog): f"got: {[r.getMessage() for r in caplog.records]}" ) + def test_binary_aggregation_counts_only_rows_that_passed_or_failed(self): + data = { + "outputs.mixed.mixed_result": ["pass", "fail", "not_applicable", None, "pass", np.nan], + "outputs.mixed.mixed": [1.0, 0.0, None, np.nan, 1.0, np.nan], + "outputs.skipped.skipped_result": ["not_applicable", None, "not_applicable", np.nan, None, np.nan], + "outputs.skipped.skipped": [None, np.nan, None, np.nan, None, np.nan], + "outputs.passing.passing_result": ["pass"] * 6, + } + + aggregation = _aggregate_metrics(pd.DataFrame(data), {}) + + assert aggregation["mixed.binary_aggregate"] == 0.67 + assert aggregation["mixed.mixed"] == pytest.approx(2 / 3) + assert aggregation["passing.binary_aggregate"] == 1.0 + # Like the mean score, the binary aggregate is left out when no row has a result. + assert "skipped.binary_aggregate" not in aggregation + assert "skipped.skipped" not in aggregation + def test_aggregate_label_defect_metrics_with_nan_in_details(self): """Test that NaN/None values in details column are properly ignored during aggregation.""" data = { diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py index 9380a53c9fdd..974b094074cb 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py @@ -377,6 +377,30 @@ def test_a_decision_judge_still_gets_tool_calls(self): state = evaluator._decision.evaluate.await_args.args[0] assert "[TOOL_CALL] fetch_weather" in state["tool_calls"] + def test_evaluate_leaves_rows_without_tool_calls_out_of_the_pass_rate(self, tmp_path): + failed_call = json.loads(json.dumps(TOOL_CALL_RESPONSE).replace("sunny", "timed out")) + rows = [{"response": TOOL_CALL_RESPONSE}, {"response": failed_call}, {"response": [OPENAI_TEXT_REPLY]}] + data = tmp_path / "rows.jsonl" + data.write_text("\n".join(json.dumps(row) for row in rows), encoding="utf-8") + evaluator = _ToolCallSuccessEvaluator(model_config=dict(CHAT_CONFIG)) + + async def flow(timeout, **kwargs): + score = 0 if "timed out" in str(kwargs["tool_calls"]) else 1 + return {"llm_output": {"score": score, "status": "completed", "reason": "r"}, **JUDGE_USAGE} + + evaluator._flow = flow + result = evaluate( + data=str(data), + evaluators={"tool_call_success": evaluator}, + evaluator_config={"tool_call_success": {"column_mapping": {"response": "${data.response}"}}}, + _use_run_submitter_client=True, + ) + + results = sorted(row["outputs.tool_call_success.tool_call_success_result"] for row in result["rows"]) + assert results == ["fail", "not_applicable", "pass"] + assert result["metrics"]["tool_call_success.tool_call_success"] == 0.5 + assert result["metrics"]["tool_call_success.binary_aggregate"] == 0.5 + @pytest.mark.parametrize( "messages", [ From 8b8fa38904061c1634be61c8b8999af448bd6db9 Mon Sep 17 00:00:00 2001 From: Ahmad Qardahji Date: Tue, 3 Nov 2026 16:15:20 -0800 Subject: [PATCH 11/14] Keep evaluator inputs in their declared order _get_all_singleton_inputs flattened the overloads' input names through a set, so an evaluator's converted inputs followed the set's order, which varies with the process's hash seed. An evaluator that sends these inputs to a Decision model unchanged, as several Microsoft Foundry catalog evaluators do, could therefore list the same fields in a different order in each process, and the Decision model could answer the same input differently. The names now keep the order in which the overloads declare them. The built-in evaluators in this package build their Decision state in a fixed order and are unchanged. --- .../azure-ai-evaluation/CHANGELOG.md | 1 + .../_evaluators/_common/_base_eval.py | 9 +- .../unittests/test_decision_model_support.py | 101 +++++++++++++++++- 3 files changed, 106 insertions(+), 5 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index 4a1bf120a6a8..1e4426a96edf 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -15,6 +15,7 @@ - `_ToolCallSuccessEvaluator` no longer scores successful tool calls 0 when a chat judge model, such as gpt-4.1 or gpt-4o, puts `score` and `status` inside `properties`. The example outputs in its prompt are now valid JSON, and `score`, `status` and `reason` are read from `properties` when the output has none at the top level. A missing score is no longer reported as 0, and any score other than 0 or 1 raises a system-classified `EvaluationException`, as `_ToolInputAccuracyEvaluator` does. - `_ToolCallSuccessEvaluator` reports a list of messages that holds no tool calls, such as one that contains only text or only MCP approval messages, as not applicable without calling the chat or Decision judge model. A `tool_calls`, `function_call` or `tool_call_id` field that is None or empty, as in OpenAI's `model_dump()` of a reply without tool calls, does not count as a tool call. Strings, text that contains `[TOOL_CALL]` or `[TOOL_RESULT]`, and messages with any other content are still sent to the judge model. - `evaluate()` computes `.binary_aggregate` over the rows that passed or failed. Rows without either result, such as not applicable or errored rows, counted as failures, so the metric disagreed with the mean score. An evaluator without any passed or failed rows has no `binary_aggregate`, as one without scores has no mean. +- An evaluator's inputs keep the order in which its `__call__` overloads declare them, instead of the order of a Python `set`, which varies with the process's hash seed. An evaluator that sends these inputs to a Decision model unchanged, as several Microsoft Foundry catalog evaluators do, could list the same fields in a different order in each process, and the Decision model could answer the same input differently. The built-in evaluators in this package already send their Decision inputs in a fixed order. ## 1.18.10 (2026-10-22) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py index db3feaa8613d..7b9e46ed9934 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py @@ -249,13 +249,14 @@ def _get_matching_overload_inputs(self, **kwargs) -> List[str]: def _get_all_singleton_inputs(self) -> List[str]: """Get a flattened list of all possible singleton inputs across all overloads. + Names keep the order in which the overloads declare them, so an evaluator's inputs, and the state of a decision + request that sends them unchanged, have the same order in every process. A set's order would vary with the + process's hash seed. + :return: Flattened list of all singleton input names. :rtype: List[str] """ - all_inputs = set() - for inputs in self._singleton_inputs: - all_inputs.update(inputs) - return list(all_inputs) + return list(dict.fromkeys(name for inputs in self._singleton_inputs for name in inputs)) def _derive_conversation_converter( self, diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index eeaad0c7bac4..f64a506e2edf 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -9,11 +9,14 @@ import json import logging import math +import os +import subprocess import sys import threading from copy import deepcopy from logging.handlers import BufferingHandler from concurrent.futures import ThreadPoolExecutor +from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock, Mock, call, patch @@ -23,7 +26,7 @@ from azure.core.pipeline.transport import AsyncHttpTransport, HttpResponse from azure.core.exceptions import ServiceRequestError -from azure.ai.evaluation import CoherenceEvaluator, FluencyEvaluator +from azure.ai.evaluation import CoherenceEvaluator, FluencyEvaluator, GroundednessEvaluator from azure.ai.evaluation._evaluators._common import EvaluatorBase, ModelEvaluatorBase, PromptyEvaluatorBase from azure.ai.evaluation._evaluators._common import _model_runners from azure.ai.evaluation._evaluators._common._decision_strategy import ( @@ -2150,3 +2153,99 @@ def test_optional_metrics_absence_does_not_change_summary_or_require_a_logger(se with patch.dict(sys.modules, {"opentelemetry.metrics": None}): summary = execution.finish("error") assert summary["attempt_count"] == 0 and summary["usage_state"] == "unknown" + + +DECLARED_ORDER = ["response", "context", "query", "tool_definitions"] + + +class PassThroughDecisionEvaluator(ModelEvaluatorBase): + """Sends its converted inputs to the decision model unchanged, as the catalog's task completion evaluator does.""" + + _supports_decision_models = True + + def _derive_singleton_inputs(self): + return [["response", "context", "query"], ["query", "response", "tool_definitions"]] + + async def _do_eval(self, eval_input): + return await DecisionEvaluationStrategy( + self._decision_model_runner, + result_key="probe", + error_target=ErrorTarget.EVALUATE, + questions={ + "status": {"type": "noul", "instructions": "Applicable?"}, + "probe": {"type": "noul", "instructions": "Good?"}, + }, + scoring=BinaryScoring(0.5), + threshold=1, + timeout=self._LLM_CALL_TIMEOUT, + ).evaluate(eval_input) + + +# Runs in a fresh interpreter, because the hash seed is fixed when the interpreter starts. +HASH_SEED_PROBE = """ +import json +import azure.ai.evaluation +from azure.ai.evaluation._evaluators._common import EvaluatorBase + + +class Probe(EvaluatorBase): + def _derive_singleton_inputs(self): + return [["response", "context", "query"], ["query", "response", "tool_definitions", "ground_truth"]] + + async def _do_eval(self, eval_input): + return {} + + +eval_input = Probe()._convert_kwargs_to_eval_input(query="q", response="r", context="c") +print(json.dumps({"file": azure.ai.evaluation.__file__, "order": list(eval_input[0])})) +""" + + +class TestDecisionStateOrder: + def test_inputs_keep_the_order_their_overloads_declare(self): + evaluator = PassThroughDecisionEvaluator(result_key="probe", model_config=DECISION_CONFIG) + assert evaluator._get_all_singleton_inputs() == DECLARED_ORDER + eval_input = evaluator._convert_kwargs_to_eval_input(query="q", response="r", tool_definitions=[{"name": "t"}]) + assert list(eval_input[0]) == DECLARED_ORDER + + def test_a_built_in_with_several_overloads_keeps_its_declared_order(self): + assert GroundednessEvaluator(CHAT_CONFIG)._get_all_singleton_inputs() == DECLARED_ORDER + + def test_a_decision_request_sends_unchanged_inputs_in_their_declared_order(self, client): + payload = decision_payload(0.9, probe={"type": "noul", "noul": 0.9}) + client.response = FakeResponse(body=json.dumps(payload).encode()) + evaluator = PassThroughDecisionEvaluator(result_key="probe", model_config=DECISION_CONFIG) + evaluator(query="q", response="r", tool_definitions=[{"name": "t"}]) + assert client.calls[0]["content"].startswith( + b'{"model":"Decision-1","state":{"response":"r","context":null,"query":"q","tool_definitions":[{"name":"t"}]},' + ) + + def test_the_order_is_the_same_for_every_hash_seed(self): + import azure.ai.evaluation + + # The children import the same package as this process, from a checkout or an installed wheel. + loaded = Path(azure.ai.evaluation.__file__).resolve() + search_path = os.pathsep.join(filter(None, [str(loaded.parents[3]), os.environ.get("PYTHONPATH")])) + children = [] + try: + for seed in range(6): + environment = dict(os.environ, PYTHONHASHSEED=str(seed), PYTHONNOUSERSITE="1", PYTHONPATH=search_path) + children.append( + subprocess.Popen( + [sys.executable, "-c", HASH_SEED_PROBE], + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + ) + for seed, child in enumerate(children): + output, errors = child.communicate(timeout=180) + assert child.returncode == 0, errors[-2000:] + result = json.loads(output.strip().splitlines()[-1]) + assert Path(result["file"]).resolve() == loaded + assert result["order"] == DECLARED_ORDER + ["ground_truth"], f"PYTHONHASHSEED={seed}" + finally: + for child in children: + if child.poll() is None: + child.kill() From abc053b6a61083f4c2238c729e459d5c988c5cc4 Mon Sep 17 00:00:00 2001 From: Ahmad Qardahji Date: Tue, 3 Nov 2026 16:23:18 -0800 Subject: [PATCH 12/14] Keep the last result column deciding binary_aggregate An evaluator with several _result columns reports its last one, as before. A last column without pass or fail rows now leaves no binary_aggregate instead of an earlier column's rate. The CHANGELOG says only pass and fail count, which also affects custom evaluators that return other values. Tests cover all-fail and boolean result columns, several result columns, and OpenAI tool calls on the Decision judge. --- .../azure-ai-evaluation/CHANGELOG.md | 2 +- .../ai/evaluation/_evaluate/_evaluate.py | 13 +++-- .../tests/unittests/test_evaluate.py | 19 ++++++++ .../test_tool_call_success_evaluator.py | 47 +++++++++++-------- 4 files changed, 56 insertions(+), 25 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index 1e4426a96edf..4f7775588b05 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -14,7 +14,7 @@ - `_ToolCallSuccessEvaluator` no longer scores successful tool calls 0 when a chat judge model, such as gpt-4.1 or gpt-4o, puts `score` and `status` inside `properties`. The example outputs in its prompt are now valid JSON, and `score`, `status` and `reason` are read from `properties` when the output has none at the top level. A missing score is no longer reported as 0, and any score other than 0 or 1 raises a system-classified `EvaluationException`, as `_ToolInputAccuracyEvaluator` does. - `_ToolCallSuccessEvaluator` reports a list of messages that holds no tool calls, such as one that contains only text or only MCP approval messages, as not applicable without calling the chat or Decision judge model. A `tool_calls`, `function_call` or `tool_call_id` field that is None or empty, as in OpenAI's `model_dump()` of a reply without tool calls, does not count as a tool call. Strings, text that contains `[TOOL_CALL]` or `[TOOL_RESULT]`, and messages with any other content are still sent to the judge model. -- `evaluate()` computes `.binary_aggregate` over the rows that passed or failed. Rows without either result, such as not applicable or errored rows, counted as failures, so the metric disagreed with the mean score. An evaluator without any passed or failed rows has no `binary_aggregate`, as one without scores has no mean. +- `evaluate()` computes `.binary_aggregate` over the rows whose result is `pass` or `fail`. Other rows, such as not applicable or errored rows, used to count as failures, although the mean score leaves them out. An evaluator without `pass` or `fail` rows, including a custom evaluator whose results have other values, has no `binary_aggregate`, as one without scores has no mean. - An evaluator's inputs keep the order in which its `__call__` overloads declare them, instead of the order of a Python `set`, which varies with the process's hash seed. An evaluator that sends these inputs to a Decision model unchanged, as several Microsoft Foundry catalog evaluators do, could list the same fields in a different order in each process, and the Decision model could answer the same input differently. The built-in evaluators in this package already send their Decision inputs in a fixed order. ## 1.18.10 (2026-10-22) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py index c8590defbb8e..61c05f398fb8 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py @@ -273,9 +273,10 @@ def _aggregation_binary_output(df: pd.DataFrame) -> Dict[str, float]: """ Aggregate binary output results (pass/fail) from evaluation dataframe. - For each evaluator, calculates the proportion of "pass" results among the rows that passed or failed. Rows - without either result, such as not applicable or errored rows, are left out, as they are from the mean score, - and an evaluator without any passed or failed rows gets no binary aggregate. + For each evaluator, calculates the proportion of "pass" results among the rows whose result is "pass" or "fail". + Other rows, such as not applicable or errored rows, are left out, as they are from the mean score, and a result + column without "pass" or "fail" rows gives no binary aggregate. An evaluator with several result columns reports + its last one. :param df: The dataframe of evaluation results. :type df: ~pandas.DataFrame @@ -316,11 +317,13 @@ def _aggregation_binary_output(df: pd.DataFrame) -> Dict[str, float]: pass_count = value_counts.get(EVALUATION_PASS_FAIL_MAPPING[True], 0) judged_rows = pass_count + value_counts.get(EVALUATION_PASS_FAIL_MAPPING[False], 0) - if judged_rows == 0: - continue # Set the result with the evaluator name as the key result_key = f"{evaluator_name}.{BINARY_AGGREGATE_SUFFIX}" + if judged_rows == 0: + # The evaluator's last result column decides, so an earlier column's rate is not kept. + results.pop(result_key, None) + continue results[result_key] = round(pass_count / judged_rows, 2) return results diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index 848484ab6d47..c26485ff1bd2 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -789,6 +789,8 @@ def test_binary_aggregation_counts_only_rows_that_passed_or_failed(self): "outputs.skipped.skipped_result": ["not_applicable", None, "not_applicable", np.nan, None, np.nan], "outputs.skipped.skipped": [None, np.nan, None, np.nan, None, np.nan], "outputs.passing.passing_result": ["pass"] * 6, + "outputs.failing.failing_result": ["fail", None, "fail", "not_applicable", np.nan, "fail"], + "outputs.custom.custom_result": [True, False, True, True, False, True], } aggregation = _aggregate_metrics(pd.DataFrame(data), {}) @@ -796,9 +798,26 @@ def test_binary_aggregation_counts_only_rows_that_passed_or_failed(self): assert aggregation["mixed.binary_aggregate"] == 0.67 assert aggregation["mixed.mixed"] == pytest.approx(2 / 3) assert aggregation["passing.binary_aggregate"] == 1.0 + assert aggregation["failing.binary_aggregate"] == 0.0 # Like the mean score, the binary aggregate is left out when no row has a result. assert "skipped.binary_aggregate" not in aggregation assert "skipped.skipped" not in aggregation + # Only "pass" and "fail" are results, so a column of other values has no binary aggregate. + assert "custom.binary_aggregate" not in aggregation + + def test_binary_aggregation_reports_the_last_result_column_of_an_evaluator(self): + data = { + "outputs.qa.f1_score_result": ["pass", "pass", "fail"], + "outputs.qa.groundedness_result": ["not_applicable", None, "not_applicable"], + "outputs.rouge.rouge_precision_result": ["not_applicable", None, "not_applicable"], + "outputs.rouge.rouge_recall_result": ["pass", "fail", None], + } + + aggregation = _aggregate_metrics(pd.DataFrame(data), {}) + + # An earlier column's rate does not stand in for a last column without results. + assert "qa.binary_aggregate" not in aggregation + assert aggregation["rouge.binary_aggregate"] == 0.5 def test_aggregate_label_defect_metrics_with_nan_in_details(self): """Test that NaN/None values in details column are properly ignored during aggregation.""" diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py index 974b094074cb..4a12ab541d24 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py @@ -57,6 +57,25 @@ "function_call": None, "tool_calls": None, } +OPENAI_TOOL_CALL_RESPONSE = [ + { + **OPENAI_TEXT_REPLY, + "content": "Checking the weather.", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "fetch_weather", "arguments": '{"city": "Paris"}'}, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [{"type": "tool_result", "tool_result": {"error": "timed out"}}], + }, + {**OPENAI_TEXT_REPLY, "content": "The weather service timed out."}, +] JUDGE_USAGE = {"input_token_count": 11, "output_token_count": 7, "total_token_count": 18} @@ -330,25 +349,7 @@ def test_a_response_without_tool_calls_is_not_applicable_without_a_judge_call(se }, {"role": "assistant", "content": [{"type": "text", "text": "Done."}]}, ], - [ - { - **OPENAI_TEXT_REPLY, - "content": "Checking the weather.", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": {"name": "fetch_weather", "arguments": '{"city": "Paris"}'}, - } - ], - }, - { - "role": "tool", - "tool_call_id": "call_1", - "content": [{"type": "tool_result", "tool_result": {"error": "timed out"}}], - }, - {**OPENAI_TEXT_REPLY, "content": "The weather service timed out."}, - ], + OPENAI_TOOL_CALL_RESPONSE, ], ids=[ "tool-call", @@ -377,6 +378,14 @@ def test_a_decision_judge_still_gets_tool_calls(self): state = evaluator._decision.evaluate.await_args.args[0] assert "[TOOL_CALL] fetch_weather" in state["tool_calls"] + def test_a_decision_judge_still_gets_openai_tool_calls(self): + evaluator = _ToolCallSuccessEvaluator(model_config=dict(DECISION_CONFIG)) + evaluator._decision.evaluate = AsyncMock(return_value={"tool_call_success": 0.0}) + + assert evaluator(response=OPENAI_TOOL_CALL_RESPONSE)["tool_call_success"] == 0.0 + assert evaluator._decision.evaluate.await_count == 1 + assert "fetch_weather" in str(evaluator._decision.evaluate.await_args.args[0]["tool_calls"]) + def test_evaluate_leaves_rows_without_tool_calls_out_of_the_pass_rate(self, tmp_path): failed_call = json.loads(json.dumps(TOOL_CALL_RESPONSE).replace("sunny", "timed out")) rows = [{"response": TOOL_CALL_RESPONSE}, {"response": failed_call}, {"response": [OPENAI_TEXT_REPLY]}] From 70135d235fb9bdbb4c343b08676aec7cbd6c7992 Mon Sep 17 00:00:00 2001 From: Ahmad Qardahji Date: Tue, 3 Nov 2026 16:41:14 -0800 Subject: [PATCH 13/14] Run the hash seed test's processes from an empty directory python -c puts the working directory first on the path, so in CI jobs that install the built wheel and run pytest from the package directory, the child processes imported the checkout instead of the installed package. They now start in an empty temporary directory, and three seeds keep the test short. --- .../tests/unittests/test_decision_model_support.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index f64a506e2edf..5761abc9e135 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -2220,26 +2220,29 @@ def test_a_decision_request_sends_unchanged_inputs_in_their_declared_order(self, b'{"model":"Decision-1","state":{"response":"r","context":null,"query":"q","tool_definitions":[{"name":"t"}]},' ) - def test_the_order_is_the_same_for_every_hash_seed(self): + def test_the_order_is_the_same_for_every_hash_seed(self, tmp_path): import azure.ai.evaluation - # The children import the same package as this process, from a checkout or an installed wheel. + # The children import the same package as this process, from a checkout or an installed wheel. They start in + # an empty directory, because "python -c" puts the working directory first on the path, so a checkout there + # would shadow an installed wheel. loaded = Path(azure.ai.evaluation.__file__).resolve() search_path = os.pathsep.join(filter(None, [str(loaded.parents[3]), os.environ.get("PYTHONPATH")])) children = [] try: - for seed in range(6): + for seed in range(1, 4): environment = dict(os.environ, PYTHONHASHSEED=str(seed), PYTHONNOUSERSITE="1", PYTHONPATH=search_path) children.append( subprocess.Popen( [sys.executable, "-c", HASH_SEED_PROBE], + cwd=tmp_path, env=environment, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, ) ) - for seed, child in enumerate(children): + for seed, child in enumerate(children, start=1): output, errors = child.communicate(timeout=180) assert child.returncode == 0, errors[-2000:] result = json.loads(output.strip().splitlines()[-1]) From 0028aa3e7880532630ed14d79f45b097d8e7d10f Mon Sep 17 00:00:00 2001 From: Ahmad Qardahji Date: Tue, 3 Nov 2026 17:56:06 -0800 Subject: [PATCH 14/14] Leave unsupported role and content type values to the judge The no-tool-call check compared role and content type values with sets, so an unhashable value, such as a list role or a dict type, raised TypeError. A value that is not a string now fails the check, so the message goes to the judge or the input validator, as it did before the check existed. --- .../_tool_call_success/_tool_call_success.py | 15 +++++++++--- .../test_tool_call_success_evaluator.py | 24 +++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py index 5a43040055ff..7ecd702a7a8b 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py @@ -317,6 +317,11 @@ def _holds_nothing(value: Any) -> bool: return value is None or (isinstance(value, (str, list, dict)) and not value) +def _is_one_of(value: Any, names: frozenset) -> bool: + """Return True when the value is one of the names; any other value, including an unhashable one, is not.""" + return isinstance(value, str) and value in names + + def _has_no_tool_calls(messages) -> bool: """Return True only for a list of text messages and MCP approvals that holds no tool calls, results or traces. @@ -332,17 +337,21 @@ def _has_no_tool_calls(messages) -> bool: content = message.get("content") items = content if isinstance(content, list): - items = [item for item in content if not (isinstance(item, dict) and item.get("type") in _APPROVAL_TYPES)] + items = [ + item + for item in content + if not (isinstance(item, dict) and _is_one_of(item.get("type"), _APPROVAL_TYPES)) + ] if content and not items: # Only MCP approval requests or responses, which are not tool calls; a response names its request # in tool_call_id. continue - if message.get("role") not in _CONVERSATION_ROLES or not _holds_nothing(message.get("tool_call_id")): + if not _is_one_of(message.get("role"), _CONVERSATION_ROLES) or not _holds_nothing(message.get("tool_call_id")): return False if isinstance(content, str): texts = [content] elif isinstance(items, list) and all( - isinstance(item, dict) and item.get("type") in _TEXT_CONTENT_TYPES for item in items + isinstance(item, dict) and _is_one_of(item.get("type"), _TEXT_CONTENT_TYPES) for item in items ): texts = [item.get("text") for item in items] else: diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py index 4a12ab541d24..82ce227c8ad9 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_tool_call_success_evaluator.py @@ -441,11 +441,35 @@ async def flow(timeout, **kwargs): [{"role": "assistant", "content": "[TOOL_CALL] fetch_weather()"}], [{"role": "assistant", "content": [{"type": "output_text", "text": "[TOOL_RESULT] ok"}]}], [{"role": "assistant", "content": [{"type": "text", "text": None}]}], + [{"role": ["assistant"], "content": "Hello."}], + [{"role": {"name": "assistant"}, "content": "Hello."}], + [{"role": "assistant", "content": [{"type": {"kind": "text"}, "text": "Hello."}]}], + [{"role": "assistant", "content": [{"type": ["text"], "text": "Hello."}]}], + [{"role": "assistant", "content": [{"type": {"kind": "mcp_approval_request"}}]}], ], ) def test_only_recognized_text_messages_count_as_having_no_tool_calls(self, messages): assert _has_no_tool_calls(messages) is False + @pytest.mark.parametrize("role", [["assistant"], {"name": "assistant"}], ids=["list", "dict"]) + def test_an_unsupported_role_value_still_goes_to_the_judge(self, role): + evaluator = judge({"score": 1, "status": "completed", "reason": "r"}) + + result = evaluator(response=[{"role": role, "content": "Hello."}]) + + assert evaluator._flow.await_count == 1 + assert result["tool_call_success_result"] == "pass" + + @pytest.mark.parametrize("item_type", [{"kind": "text"}, ["text"]], ids=["dict", "list"]) + def test_an_unsupported_content_type_value_gets_the_input_validation_error(self, item_type): + evaluator = judge({"score": 1, "status": "completed", "reason": "r"}) + + with pytest.raises(EvaluationException, match="Invalid content type") as raised: + evaluator(response=[{"role": "assistant", "content": [{"type": item_type, "text": "Hello."}]}]) + + assert raised.value.blame == ErrorBlame.USER_ERROR + assert evaluator._flow.await_count == 0 + def test_text_messages_have_no_tool_calls(self): assert _has_no_tool_calls(TEXT_RESPONSE) is True assert _has_no_tool_calls([{"role": "assistant", "content": []}]) is True