From dbcca62c664e500cca8fc563d3d7662e2c59b728 Mon Sep 17 00:00:00 2001 From: Robin Date: Sat, 5 Sep 2026 21:37:04 -0700 Subject: [PATCH] group scorer feature --- agentx/evaluations/client.py | 19 ++++++- agentx/evaluations/models.py | 7 +++ agentx/evaluations/runner.py | 22 ++++++++- agentx/monitor/client.py | 5 ++ agentx/monitor/judge_scorers.py | 5 ++ agentx/monitor/scorer_groups.py | 88 +++++++++++++++++++++++++++++++++ tests/test_judge_scorers.py | 31 ++++++++++++ tests/test_multi_judge.py | 76 ++++++++++++++++++++++++++++ 8 files changed, 250 insertions(+), 3 deletions(-) create mode 100644 agentx/monitor/scorer_groups.py create mode 100644 tests/test_multi_judge.py diff --git a/agentx/evaluations/client.py b/agentx/evaluations/client.py index 5dcef56..6081fa4 100644 --- a/agentx/evaluations/client.py +++ b/agentx/evaluations/client.py @@ -314,10 +314,15 @@ def init_run( scorer_id: Optional[str] = None, evaluation_settings_id: Optional[str] = None, split: Optional[str] = None, + additional_scorer_ids: Optional[List[str]] = None, + scorer_group_id: Optional[str] = None, ) -> EvaluationRun: """``scorer_id`` names the LLM Judge Scorer grading this run (its id doubles as the wire's ``evaluationSettingsId``). ``evaluation_settings_id`` is the pre-consolidation - alias and keeps working. ``split`` records the named case subset this run covers.""" + alias and keeps working. ``split`` records the named case subset this run covers. + ``additional_scorer_ids`` (self-host): extra judge scorers that each pass their own + verdict on every result from the same single agent execution - verdicts land in each + result row's ``judgeScorerResults`` and the run's ``scorerBreakdown``.""" from agentx.version import VERSION grader_id = _resolve_scorer_id(scorer_id, evaluation_settings_id) @@ -335,6 +340,12 @@ def init_run( } if grader_id: payload["evaluationSettingsId"] = grader_id + if additional_scorer_ids: + payload["additionalScorerIds"] = additional_scorer_ids + # Scorer group grading (self-host): the group's weighted 0-10 aggregate fills the rating + # column and member verdicts land per row. Mutually exclusive with scorer_id (group wins). + if scorer_group_id: + payload["scorerGroupId"] = scorer_group_id if split: payload["split"] = split data = self._request("POST", "/runs", json=self._with_workspace(payload)) @@ -372,6 +383,7 @@ def gate_run( tolerance: Optional[float] = None, record: bool = True, caller: Optional[str] = "sdk", + scorer: Optional[str] = None, ) -> Dict[str, Any]: # CI gate (self-host): pass/fail a finalized run against an absolute rating floor and/or # the dataset's previous completed run. Recorded into gate history by default (the @@ -389,6 +401,11 @@ def gate_run( params["record"] = "true" if caller: params["caller"] = caller + # Multi-judge runs (self-host): gate a named additional scorer (id or name, e.g. + # scorer="Safety") instead of the primary - failUnder/noRegression then use that + # scorer's own per-result verdicts. Unknown names are a hard 400 from the engine. + if scorer: + params["scorer"] = scorer return self._request("GET", f"/runs/{run_id}/gate", params=params) def analyze_run( diff --git a/agentx/evaluations/models.py b/agentx/evaluations/models.py index 72b1a1e..76d33ad 100644 --- a/agentx/evaluations/models.py +++ b/agentx/evaluations/models.py @@ -75,6 +75,9 @@ class Dataset(BaseModel): rejection_criteria: Optional[str] = Field(default=None, alias="rejectionCriteria") evaluation_criteria: Optional[str] = Field(default=None, alias="evaluationCriteria") questions: List[DatasetQuestion] = Field(default_factory=list) + # Custom code scorers attached to this dataset - [{ id, name, code, enabled }]. Retrievable, + # so a fetched dataset round-trips them (import_dataset copies them to the new dataset). + code_scorers: Optional[List[Dict[str, Any]]] = Field(default=None, alias="codeScorers") status: str = "published" version_id: Optional[str] = Field(default=None, alias="versionId") # Sovereignty & Portability - models selected to compare on this dataset. @@ -108,6 +111,8 @@ class EvaluationSettings(BaseModel): acceptance_criteria: Optional[str] = Field(default=None, alias="acceptanceCriteria") rejection_criteria: Optional[str] = Field(default=None, alias="rejectionCriteria") evaluation_criteria: Optional[str] = Field(default=None, alias="evaluationCriteria") + # Custom code scorers on this scorer's offline profile - [{ id, name, code, enabled }]. + code_scorers: Optional[List[Dict[str, Any]]] = Field(default=None, alias="codeScorers") # LLM-as-judge overrides. None means "use the server default" (raw prompt template / OpenAI # gpt-5.5). See client.evaluations.settings.builder(judge_prompt=..., judge_model=...). judge_prompt: Optional[str] = Field(default=None, alias="judgePrompt") @@ -435,6 +440,8 @@ class RunResultRow(BaseModel): bleu_score: Optional[float] = Field(default=None, alias="bleuScore") rouge_score: Optional[float] = Field(default=None, alias="rougeScore") code_scorer_results: Optional[List[Dict[str, Any]]] = Field(default=None, alias="codeScorerResults") + # Verdicts from the run's ADDITIONAL judge scorers: [{scorerId, name, rating, justification}]. + judge_scorer_results: Optional[List[Dict[str, Any]]] = Field(default=None, alias="judgeScorerResults") raw: Dict[str, Any] = Field(default_factory=dict) class Config: diff --git a/agentx/evaluations/runner.py b/agentx/evaluations/runner.py index 51fbc57..7942dbc 100644 --- a/agentx/evaluations/runner.py +++ b/agentx/evaluations/runner.py @@ -77,6 +77,9 @@ def __init__(self, data: Dict[str, Any]): self.baseline_average: Optional[float] = data.get("baselineAverage") self.baseline_run_id: Optional[str] = data.get("baselineRunId") self.checks: List[Dict[str, Any]] = data.get("checks", []) + # Multi-judge runs: {"id", "name"} of the additional scorer being gated when the gate + # ran with scorer=..., None when gating the primary. + self.gated_scorer: Optional[Dict[str, Any]] = data.get("gatedScorer") @property def exit_code(self) -> int: @@ -354,13 +357,16 @@ def gate( no_regression: bool = False, tolerance: Optional[float] = None, caller: str = "sdk", + scorer: Optional[str] = None, ) -> "GateResult": """CI gate (self-host): pass/fail this finalized run so a CI job can block a merge. ``fail_under`` fails the gate when the run's average rating is below the floor; ``no_regression=True`` fails it when the average dropped more than ``tolerance`` (default 0.5, judge scores are noisy) below the dataset's previous completed run. - At least one check is required. Prints a CI-log-friendly verdict and returns a + At least one check is required. On a multi-judge run, ``scorer`` (an additional + scorer's id or name, e.g. ``scorer="Safety"``) gates that scorer's own average + instead of the primary's - "fail if Safety is low even when the average looks fine". Prints a CI-log-friendly verdict and returns a :class:`GateResult` - the caller decides the exit code:: report = client.evaluations.run(...).execute(my_agent).finalize() @@ -374,6 +380,7 @@ def gate( no_regression=no_regression, tolerance=tolerance, caller=caller, + scorer=scorer, ) result = GateResult(data) _say() @@ -589,6 +596,7 @@ def gate_run( tolerance: Optional[float] = None, record: bool = True, caller: Optional[str] = "sdk", + scorer: Optional[str] = None, ) -> GateResult: """CI-gate any finalized run by id - the standalone form of ``EvaluationRunContext.gate()``, for gating a run created elsewhere or @@ -603,6 +611,7 @@ def gate_run( tolerance=tolerance, record=record, caller=caller, + scorer=scorer, ) ) @@ -613,6 +622,8 @@ def run( scorer_id: Optional[str] = None, evaluation_settings_id: Optional[str] = None, split: Optional[str] = None, + additional_scorer_ids: Optional[List[str]] = None, + scorer_group_id: Optional[str] = None, ) -> EvaluationRunContext: """Start a run of ``dataset_id`` against ``subject``. Pass ``scorer_id`` (an LLM Judge Scorer's id, e.g. from ``client.monitor.judge_scorers``) to grade with a specific @@ -632,7 +643,14 @@ def run( evaluation_settings = ( self._client.get_evaluation_settings(grader_id) if grader_id else None ) - run = self._client.init_run(dataset_id, subject, scorer_id=grader_id, split=split) + run = self._client.init_run( + dataset_id, + subject, + scorer_id=grader_id, + split=split, + additional_scorer_ids=additional_scorer_ids, + scorer_group_id=scorer_group_id, + ) case_count = ( sum(1 for q in dataset.questions if split in (q.main_question.splits or [])) if split diff --git a/agentx/monitor/client.py b/agentx/monitor/client.py index 6eab3fe..906d2aa 100644 --- a/agentx/monitor/client.py +++ b/agentx/monitor/client.py @@ -147,6 +147,11 @@ def __init__( # surface that matches the product; evaluations.settings and online_evaluators below # remain as its profile-level views. self.judge_scorers = JudgeScorersClient(api_key=api_key, base_url=self._api_root()) + from agentx.monitor.scorer_groups import ScorerGroupsClient + + # Scorer groups: mixed-kind scorers composed into one 0-10 score (weights + must-pass + # gates) - a group grades dataset runs (scorer_group_id) and, when online, live traffic. + self.scorer_groups = ScorerGroupsClient(api_key=api_key, base_url=self._api_root()) from agentx.monitor.improvement_groups import ImprovementGroupsClient # Auto-improve: confirmed production failures -> improvement report -> code fix (via diff --git a/agentx/monitor/judge_scorers.py b/agentx/monitor/judge_scorers.py index f479f25..d73994a 100644 --- a/agentx/monitor/judge_scorers.py +++ b/agentx/monitor/judge_scorers.py @@ -36,6 +36,11 @@ def judge(self) -> Dict[str, Any]: def offline(self) -> Dict[str, Any]: return self.get("offline", {}) + @property + def code_scorers(self) -> List[Dict[str, Any]]: + """Custom code scorers on the offline profile - [{ id, name, code, enabled }].""" + return list(self.offline.get("codeScorers") or []) + @property def online(self) -> Optional[Dict[str, Any]]: return self.get("online") diff --git a/agentx/monitor/scorer_groups.py b/agentx/monitor/scorer_groups.py new file mode 100644 index 0000000..4593714 --- /dev/null +++ b/agentx/monitor/scorer_groups.py @@ -0,0 +1,88 @@ +"""Scorer groups (self-host): scorers of any kind - LLM judges, patterns, custom code/external +scorers - composed into ONE 0-10 score via per-member weights and optional must-pass gates. +Members are references: ``{"kind": "judge" | "pattern" | "custom", "refId": ..., "weight": ..., +"gate": ...}``. Grade a dataset run with a group by passing its id as ``scorer_group_id`` to +``client.evaluations.run(...)``; give it an ``online`` profile to score sampled live traffic and +raise Signals below the alert threshold.""" + +from typing import Any, Dict, List, Optional + +import requests + + +class AgentXScorerGroupsError(Exception): + pass + + +class ScorerGroup(dict): + """Wire object (dict subclass so unknown fields round-trip).""" + + @property + def id(self) -> str: + return self["_id"] + + @property + def name(self) -> str: + return self["name"] + + @property + def members(self) -> List[Dict[str, Any]]: + return list(self.get("members") or []) + + @property + def online(self) -> Optional[Dict[str, Any]]: + return self.get("online") + + +class ScorerGroupsClient: + def __init__(self, api_key: str, base_url: str): + self._api_key = api_key + self._base = base_url.rstrip("/") + "/agent-monitoring/scorer-groups" + + def _request(self, method: str, url: str, json: Optional[Dict[str, Any]] = None) -> Any: + response = requests.request( + method, + url, + headers={"x-api-key": self._api_key, "content-type": "application/json"}, + json=json, + timeout=30, + ) + if response.status_code >= 400: + raise AgentXScorerGroupsError(f"HTTP {response.status_code}: {response.text}") + return response.json() + + def list(self) -> List[ScorerGroup]: + return [ScorerGroup(g) for g in self._request("GET", self._base).get("scorerGroups", [])] + + def get(self, group_id: str) -> ScorerGroup: + return ScorerGroup(self._request("GET", f"{self._base}/{group_id}")["scorerGroup"]) + + def create( + self, + name: str, + members: List[Dict[str, Any]], + description: Optional[str] = None, + online: Optional[Dict[str, Any]] = None, + ) -> ScorerGroup: + """``members``: [{"kind": "judge"|"pattern"|"custom", "refId": ..., "weight": 1, "gate": False}]. + ``online``: {"enabled": True, "sampleRate": 0.1, "alertThreshold": 5, "severity": "medium"} + or None for offline-only.""" + payload: Dict[str, Any] = {"name": name, "members": members} + if description is not None: + payload["description"] = description + if online is not None: + payload["online"] = online + return ScorerGroup(self._request("POST", self._base, json=payload)["scorerGroup"]) + + def update(self, group_id: str, **fields: Any) -> ScorerGroup: + """Sparse update - pass any of name/description/members/online (online=None detaches + live scoring).""" + return ScorerGroup(self._request("PUT", f"{self._base}/{group_id}", json=fields)["scorerGroup"]) + + def delete(self, group_id: str) -> None: + self._request("DELETE", f"{self._base}/{group_id}") + + def ratings(self, group_id: str, window: str = "7d") -> Dict[str, Any]: + """Live score history for a group - ``{"window", "points": [{ts, averageRating, count}]}``, + the same shape online-evaluator ratings use. ``window``: "24h" | "7d" | "30d".""" + return self._request("GET", f"{self._base}/{group_id}/ratings?window={window}") diff --git a/tests/test_judge_scorers.py b/tests/test_judge_scorers.py index d4a0c52..692db0c 100644 --- a/tests/test_judge_scorers.py +++ b/tests/test_judge_scorers.py @@ -268,3 +268,34 @@ def fake_request(method, path, json=None, timeout=60): assert captured["validate"]["window"] == "24h" assert "criteria" not in captured["validate"] assert captured["publish"]["acceptanceCriteria"] == "a" + + +def test_code_scorers_are_retrievable_from_the_wire_object(): + """The wire rows may lack ids (SDK-created scorers) - retrieval must hand them back as-is.""" + from agentx.monitor.judge_scorers import JudgeScorer + + scorer = JudgeScorer( + { + "_id": "s1", + "name": "Blend", + "offline": {"codeScorers": [{"name": "Final score", "code": "return 1;", "enabled": True}]}, + } + ) + assert scorer.code_scorers == [{"name": "Final score", "code": "return 1;", "enabled": True}] + # And an offline profile without any stays an empty list, not a KeyError. + assert JudgeScorer({"_id": "s2", "name": "Plain", "offline": {}}).code_scorers == [] + + +def test_dataset_model_round_trips_code_scorers(): + """extra="ignore" used to silently drop codeScorers on read - import_dataset lost them.""" + from agentx.evaluations.models import Dataset + + wire = { + "_id": "d1", + "name": "Guarded", + "questions": [], + "codeScorers": [{"id": "cs1", "name": "gate", "code": "return 0;", "enabled": True}], + } + parsed = Dataset(**wire) + assert parsed.code_scorers == wire["codeScorers"] + assert parsed.model_dump(by_alias=True)["codeScorers"] == wire["codeScorers"] diff --git a/tests/test_multi_judge.py b/tests/test_multi_judge.py new file mode 100644 index 0000000..1251d28 --- /dev/null +++ b/tests/test_multi_judge.py @@ -0,0 +1,76 @@ +"""Wire-level tests for multi-judge dataset runs: additional_scorer_ids on run creation and +the named-scorer CI gate (gate_run(scorer=...)). The session/HTTP layer is monkeypatched; the +engine-side behavior is pinned by the engine's multiJudge.integration.test.ts.""" + +from typing import Any, Dict, List + +import pytest + +from agentx.evaluations.client import EvaluationsClient +from agentx.evaluations.models import EvaluationSubject + + +class FakeResponse: + def __init__(self, payload: Dict[str, Any], status_code: int = 200): + self._payload = payload + self.status_code = status_code + self.ok = status_code < 400 + self.text = "x" + + def json(self) -> Dict[str, Any]: + return self._payload + + +@pytest.fixture() +def recorded(monkeypatch): + calls: List[Dict[str, Any]] = [] + + def fake_request(method, url, timeout=None, **kwargs): + calls.append({"method": method, "url": url, **kwargs}) + return FakeResponse( + { + "runId": "r1", + "datasetId": "ds1", + "status": "in_progress", + "passed": True, + "checks": [], + "gatedScorer": None, + } + ) + + client = EvaluationsClient(api_key="k", base_url="http://engine:4700/api/v1") + monkeypatch.setattr(client._session, "request", fake_request) + return client, calls + + +def test_init_run_sends_additional_scorer_ids_camel_case(recorded): + client, calls = recorded + client.init_run( + "ds1", + EvaluationSubject(kind="custom_agent"), + scorer_id="primary", + additional_scorer_ids=["safety", "tone"], + ) + payload = calls[0]["json"] + assert payload["evaluationSettingsId"] == "primary" + assert payload["additionalScorerIds"] == ["safety", "tone"] + + +def test_init_run_omits_the_key_when_no_additional_scorers(recorded): + client, calls = recorded + client.init_run("ds1", EvaluationSubject(kind="custom_agent"), scorer_id="primary") + assert "additionalScorerIds" not in calls[0]["json"] + + +def test_gate_run_forwards_the_named_scorer(recorded): + client, calls = recorded + client.gate_run("r1", fail_under=5, scorer="Safety") + params = calls[0]["params"] + assert params["failUnder"] == 5 + assert params["scorer"] == "Safety" + + +def test_gate_run_leaves_scorer_off_for_primary_gates(recorded): + client, calls = recorded + client.gate_run("r1", fail_under=5) + assert "scorer" not in calls[0]["params"]