From 2747e762a744b9ee058d7ef44f0b1ac36846f4d8 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 6 Sep 2026 13:09:18 -0700 Subject: [PATCH] group scorer with session --- EVALUATIONS.md | 6 +++--- agentx/evaluations/datasets.py | 2 +- agentx/evaluations/evaluation_settings.py | 2 +- agentx/evaluations/models.py | 4 ++-- agentx/evaluations/runner.py | 13 +++++-------- agentx/monitor/client.py | 18 ++++++++++++++++++ agentx/monitor/scorer_groups.py | 4 +++- agentx/monitor/sessions.py | 12 ++++++++++++ tests/test_integrations.py | 4 +++- tests/test_selfhost_analysis_fallback.py | 4 ++-- 10 files changed, 50 insertions(+), 19 deletions(-) diff --git a/EVALUATIONS.md b/EVALUATIONS.md index c59bf4e..057aac2 100644 --- a/EVALUATIONS.md +++ b/EVALUATIONS.md @@ -432,7 +432,7 @@ Score strictly: any missing policy detail is a failing response.""", ``` - `judge_prompt` is a raw template. `{input}`, `{output}`, and `{expected}` are substituted in; everything else (chain of thought, capabilities/references, criteria, per-question `judge_guideline`, delegation notes) is appended automatically after it, so a custom prompt can restructure the grading philosophy without ever losing that context. Omit it to keep the default rubric. -- `judge_model` accepts any OpenAI or Anthropic model id (`client.evaluations.list_models(provider="Anthropic")` to discover valid ones). Omit it to keep the default (`gpt-5.5`). +- `judge_model` accepts any OpenAI or Anthropic model id (`client.evaluations.list_models(provider="Anthropic")` to discover valid ones). Omit it to keep the engine default (`gpt-5.6-luna`). - `list_models()` is **hosted platform only**: it calls the hosted API's `/custom-agent-evaluations/models` registry, which the self-host engine does not serve (404, surfaced as `AgentXEvaluationsError`). On self-host, pass any model id your engine's judge keys can reach. --- @@ -902,7 +902,7 @@ This run + gate flow is the **self-host CI path**. The separate CI-runs API in [ report = client.evaluations.run(...).execute(my_agent).finalize().analyze( mode="auto", # "auto" | "sync" | "batch" quality_mode="quality_first", # "quality_first" | "balanced" - judges=["gpt-5.5", "claude-opus-4-8"], # 1-3 model ids; omit for a single gpt-5.5 judge + judges=["gpt-5.6-luna", "claude-opus-4-8"], # 1-3 model ids; omit for the platform default judge ) report.summary # str | None, overall narrative summary @@ -933,7 +933,7 @@ This is separate from, and available even without, the numeric `average_rating`/ |---|---|---| | `mode` | `"auto"` (default), `"sync"`, `"batch"` | How item scoring executes server-side; `"auto"` picks based on run size | | `quality_mode` | `"quality_first"`, `"balanced"` | `"quality_first"` runs a second judge on every item; `"balanced"` samples based on risk | -| `judges` | 1-3 model ids | Which LLM(s) score each response. The first always runs; a second confirms, a third only breaks a tie between the first two. Defaults to a single judge, `["gpt-5.5"]`, if omitted. | +| `judges` | 1-3 model ids | Which LLM(s) score each response. The first always runs; a second confirms, a third only breaks a tie between the first two. Omit to score with a single judge, the engine's platform default model. | | `poll_interval` | seconds, default `5.0` | How often to check job status while waiting | | `timeout` | seconds, default `1800.0` | Give up waiting after this long (the job keeps running server-side; call `get_report()` later to check on it) | diff --git a/agentx/evaluations/datasets.py b/agentx/evaluations/datasets.py index b0f5189..d650700 100644 --- a/agentx/evaluations/datasets.py +++ b/agentx/evaluations/datasets.py @@ -48,7 +48,7 @@ def __init__( "questions": [], } # LLM-as-judge overrides for this dataset's own grading config. Omit either to keep the - # server default (raw prompt template / OpenAI gpt-5.5, see EVALUATIONS.md). judge_model + # server default (raw prompt template / gpt-5.6-luna, see EVALUATIONS.md). judge_model # must be one of client.evaluations.list_models() (OpenAI or Anthropic). if judge_prompt is not None: self._payload["judgePrompt"] = judge_prompt diff --git a/agentx/evaluations/evaluation_settings.py b/agentx/evaluations/evaluation_settings.py index 54c930c..651cfe2 100644 --- a/agentx/evaluations/evaluation_settings.py +++ b/agentx/evaluations/evaluation_settings.py @@ -45,7 +45,7 @@ def __init__( "evaluationCriteria": evaluation_criteria, } # LLM-as-judge overrides. Omit either to keep the server default (raw prompt template / - # OpenAI gpt-5.5, see EVALUATIONS.md). judge_model must be one of + # gpt-5.6-luna, see EVALUATIONS.md). judge_model must be one of # client.evaluations.list_models() (OpenAI or Anthropic). if judge_prompt is not None: self._payload["judgePrompt"] = judge_prompt diff --git a/agentx/evaluations/models.py b/agentx/evaluations/models.py index 76d33ad..d2f69e2 100644 --- a/agentx/evaluations/models.py +++ b/agentx/evaluations/models.py @@ -113,8 +113,8 @@ class EvaluationSettings(BaseModel): evaluation_criteria: Optional[str] = Field(default=None, alias="evaluationCriteria") # Custom code scorers on this scorer's offline profile - [{ id, name, code, enabled }]. code_scorers: Optional[List[Dict[str, Any]]] = Field(default=None, alias="codeScorers") - # LLM-as-judge overrides. None means "use the server default" (raw prompt template / OpenAI - # gpt-5.5). See client.evaluations.settings.builder(judge_prompt=..., judge_model=...). + # LLM-as-judge overrides. None means "use the server default" (raw prompt template / + # gpt-5.6-luna). See client.evaluations.settings.builder(judge_prompt=..., judge_model=...). judge_prompt: Optional[str] = Field(default=None, alias="judgePrompt") judge_model: Optional[str] = Field(default=None, alias="judgeModel") status: str = "published" diff --git a/agentx/evaluations/runner.py b/agentx/evaluations/runner.py index 7942dbc..26d2a23 100644 --- a/agentx/evaluations/runner.py +++ b/agentx/evaluations/runner.py @@ -63,9 +63,6 @@ def _say(*args, **kwargs) -> None: "l4_final_reduce": "writing final report", } -_DEFAULT_JUDGE_MODEL = "gpt-5.5" - - class GateResult: """Wire result of the CI gate (GET /runs/:id/gate) with attribute access for the fields a CI script actually branches on.""" @@ -456,16 +453,16 @@ def analyze( Args: mode: "auto" (default), "sync", or "batch" - how item scoring executes server-side. quality_mode: "quality_first" or "balanced" - how many items get a second/third judge. - judges: 1-3 model ids, e.g. ``["gpt-5.5", "claude-opus-4-8"]``. Defaults to a single - judge, ``["gpt-5.5"]``, rather than the dashboard's 3-judge default - SDK runs are - typically lighter-weight, quick-start evaluations. + judges: 1-3 model ids from ``client.evaluations.list_models()``, e.g. + ``["gpt-5.6-luna", "claude-opus-4-8"]``. Omit to let the engine score with its + platform default model (a single judge, rather than the dashboard's 3-judge + default - SDK runs are typically lighter-weight, quick-start evaluations). poll_interval: seconds between status checks while waiting. timeout: give up waiting after this many seconds (the job keeps running server-side; call ``get_report()`` later to check on it). """ if judges is not None and not (1 <= len(judges) <= 3): raise ValueError("judges must contain 1-3 model ids") - resolved_judges = judges if judges is not None else [_DEFAULT_JUDGE_MODEL] _say() with Spinner("Analyzing - AI is reviewing your results") as spinner: @@ -474,7 +471,7 @@ def analyze( self._run.run_id, mode=mode, quality_mode=quality_mode, - judges=resolved_judges, + judges=judges, ) deadline = time.monotonic() + timeout status = self._client.get_analysis_status(self._run.run_id) diff --git a/agentx/monitor/client.py b/agentx/monitor/client.py index 906d2aa..16acf29 100644 --- a/agentx/monitor/client.py +++ b/agentx/monitor/client.py @@ -429,6 +429,24 @@ def list_session_spans(self, session_id: str) -> List[dict]: data = self._request("GET", f"/ingest/sessions/{session_id}/spans", base=self._api_root()) return data.get("spans", []) if isinstance(data, dict) else data + def list_session_scores(self, session_id: str) -> List[dict]: + """Every session-level verdict on the session, newest first: session-scoped online + evaluators (kind ``online-eval:``), session-scoped scorer groups + (``scorer-group:``), and legacy coherence rows.""" + data = self._request( + "GET", f"/agent-monitoring/sessions/{session_id}/scores", base=self._api_root() + ) + return data.get("scores", []) if isinstance(data, dict) else data + + def run_session_sweep(self) -> dict: + """Run the idle-session sweep once, now - the tick that scores quiet multi-turn + sessions with every enabled session-scoped evaluator and scorer group. Production + engines run this automatically every minute; the manual trigger exists for demos, + tests, and backfills. Returns ``{"judged": n}``.""" + return self._request( + "POST", "/agent-monitoring/session-sweep/run", base=self._api_root(), timeout=300 + ) + # ------------------------------------------------------------------ # Model portability (self-host): replay a trace's input against other models # ------------------------------------------------------------------ diff --git a/agentx/monitor/scorer_groups.py b/agentx/monitor/scorer_groups.py index 4593714..69f22e1 100644 --- a/agentx/monitor/scorer_groups.py +++ b/agentx/monitor/scorer_groups.py @@ -65,7 +65,9 @@ def create( online: Optional[Dict[str, Any]] = None, ) -> ScorerGroup: """``members``: [{"kind": "judge"|"pattern"|"custom", "refId": ..., "weight": 1, "gate": False}]. - ``online``: {"enabled": True, "sampleRate": 0.1, "alertThreshold": 5, "severity": "medium"} + ``online``: {"enabled": True, "sampleRate": 0.1, "alertThreshold": 5, "severity": "medium"}. + Add ``"scope": "session", "idleSeconds": 120`` to score whole multi-turn sessions once + idle, instead of each sampled trace. or None for offline-only.""" payload: Dict[str, Any] = {"name": name, "members": members} if description is not None: diff --git a/agentx/monitor/sessions.py b/agentx/monitor/sessions.py index 0d0e76e..e633451 100644 --- a/agentx/monitor/sessions.py +++ b/agentx/monitor/sessions.py @@ -19,3 +19,15 @@ def coherence_check(self, session_id: str) -> dict: def spans(self, session_id: str) -> List[dict]: """Every span in the session (roots and children), oldest first.""" return self._client.list_session_spans(session_id) + + def scores(self, session_id: str) -> List[dict]: + """Session-level verdicts, newest first. ``kind`` says who scored: a session-scoped + online evaluator (``online-eval:``) or a session-scoped scorer group + (``scorer-group:``).""" + return self._client.list_session_scores(session_id) + + def run_sweep(self) -> dict: + """Trigger the idle-session sweep once (normally automatic, every minute) - scores + idle multi-turn sessions with every enabled session-scoped evaluator and scorer + group. Returns ``{"judged": n}``.""" + return self._client.run_session_sweep() diff --git a/tests/test_integrations.py b/tests/test_integrations.py index 909267e..0f8ed50 100644 --- a/tests/test_integrations.py +++ b/tests/test_integrations.py @@ -599,7 +599,9 @@ def test_crewai_captures_real_per_task_timing_via_event_bus(): durations are provably unequal — the old "divide latency evenly across tasks" approximation would have reported them as identical. """ - crewai = pytest.importorskip("crewai") + # exc_type=Exception: on older Pythons crewai can raise TypeError (PEP 604 syntax) at + # import time, and a broken optional integration should skip this test, not fail it. + crewai = pytest.importorskip("crewai", exc_type=Exception) from crewai.events.event_bus import crewai_event_bus from crewai.events.types.task_events import TaskCompletedEvent, TaskStartedEvent from crewai.tasks.task_output import TaskOutput diff --git a/tests/test_selfhost_analysis_fallback.py b/tests/test_selfhost_analysis_fallback.py index 1720bf4..fa48c36 100644 --- a/tests/test_selfhost_analysis_fallback.py +++ b/tests/test_selfhost_analysis_fallback.py @@ -210,12 +210,12 @@ def test_the_fallback_request_gets_the_long_analysis_timeout(): {("POST", f"{API_ROOT}/evaluate/analyze/{RUN}"): FakeResponse(200, {"status": "completed"})} ) - client.analyze_run(RUN, judges=["gpt-5.5"]) + client.analyze_run(RUN, judges=["gpt-5.6-luna"]) method, url, kwargs = session.calls[-1] assert url == f"{API_ROOT}/evaluate/analyze/{RUN}" assert kwargs["timeout"] > 60, "a synchronous judge pass needs more than the 30s default" - assert kwargs["json"]["judges"] == [{"model": "gpt-5.5"}] + assert kwargs["json"]["judges"] == [{"model": "gpt-5.6-luna"}] # ---------------------------------------------------------------------------