AgentEvaluator.evaluate() fails a lower-is-better metric whose evaluator reports PASSED #7414

from . import agent

from __future__ import annotations

from google.adk.agents.llm_agent import LlmAgent
from google.adk.evaluation.eval_metrics import (
    EvalMetric,
    EvalStatus,
    Interval,
    MetricInfo,
    MetricValueInfo,
)
from google.adk.evaluation.evaluator import EvaluationResult, Evaluator, PerInvocationResult
from google.adk.evaluation.metric_evaluator_registry import DEFAULT_METRIC_EVALUATOR_REGISTRY
from google.adk.models.base_llm import BaseLlm
from google.adk.models.llm_response import LlmResponse
from google.genai import types as genai_types

METRIC_NAME = "error_rate"  # lower is better; 0.10 against a 0.50 limit is a PASS
LIMIT = 0.5
VALUE = 0.1


class ErrorRateEvaluator(Evaluator):
    def __init__(self, *, eval_metric: EvalMetric) -> None:
        self._eval_metric = eval_metric

    def evaluate_invocations(
        self, actual_invocations, expected_invocations=None, conversation_scenario=None
    ) -> EvaluationResult:
        results = [
            PerInvocationResult(
                actual_invocation=inv,
                score=VALUE,
                # The evaluator's own verdict: lower is better, 0.1 <= 0.5.
                eval_status=EvalStatus.PASSED if VALUE <= LIMIT else EvalStatus.FAILED,
            )
            for inv in actual_invocations
        ]
        return EvaluationResult(
            overall_score=VALUE,
            overall_eval_status=EvalStatus.PASSED,
            per_invocation_results=results,
        )


DEFAULT_METRIC_EVALUATOR_REGISTRY.register_evaluator(
    metric_info=MetricInfo(
        metric_name=METRIC_NAME,
        description="Toy lower-is-better metric.",
        metric_value_info=MetricValueInfo(interval=Interval(min_value=0.0, max_value=1.0)),
    ),
    evaluator=ErrorRateEvaluator,
)


class _FakeLlm(BaseLlm):
    model: str = "fake-model"

    @classmethod
    def supported_models(cls) -> list[str]:
        return ["fake-model"]

    async def generate_content_async(self, llm_request, stream: bool = False):
        yield LlmResponse(
            content=genai_types.Content(parts=[genai_types.Part(text="ok")], role="model")
        )


root_agent = LlmAgent(name="toy_agent", model=_FakeLlm(), instruction="Answer briefly.")

from __future__ import annotations

import asyncio
import json
from pathlib import Path

from google.adk.evaluation.eval_case import EvalCase, Invocation
from google.adk.evaluation.eval_set import EvalSet
from google.genai import types as genai_types

case = EvalCase(
    eval_id="case_1",
    conversation=[
        Invocation(user_content=genai_types.Content(parts=[genai_types.Part(text="hi")], role="user"))
    ],
)
Path("eval_set.json").write_text(EvalSet(eval_set_id="s", eval_cases=[case]).model_dump_json())
Path("test_config.json").write_text(json.dumps({"criteria": {"error_rate": 0.5}}))


async def main() -> None:
    from google.adk.evaluation.agent_evaluator import AgentEvaluator

    await AgentEvaluator.evaluate(
        agent_module="lowerbetter",
        eval_dataset_file_path_or_dir="eval_set.json",
        num_runs=1,
        print_detailed_results=True,
    )


asyncio.run(main())