diff --git a/agentx/monitor/client.py b/agentx/monitor/client.py index 0b2ed57..6eab3fe 100644 --- a/agentx/monitor/client.py +++ b/agentx/monitor/client.py @@ -67,6 +67,22 @@ def reported_count(self) -> int: def review_label_count(self) -> int: return int(self.get("reviewLabelCount") or 0) + @property + def alpha(self): + """Chance-corrected agreement (Krippendorff's alpha over the binary verdict pair) - + the raw ``agreement_rate`` corrected for what a weighted coin would score on this + label mix. ``None`` below the server's sample floor (``alphaMinItems`` labeled pairs) + or when every label is identical: withheld, never fabricated. 1 = perfect, 0 = no + better than chance, negative = systematically opposed.""" + return self.get("alpha") + + @property + def alpha_band(self): + """Human-readable band for ``alpha`` (poor/slight/fair/moderate/substantial/ + near-perfect), computed server-side so every surface reads the same alpha the + same way.""" + return self.get("alphaBand") + class MonitorClient: """Low-level HTTP client for the Monitor API (``/monitor``). Accessed via @@ -131,6 +147,11 @@ def __init__( # surface that matches the product; evaluations.settings and online_evaluators below # remain as its profile-level views. self.judge_scorers = JudgeScorersClient(api_key=api_key, base_url=self._api_root()) + from agentx.monitor.improvement_groups import ImprovementGroupsClient + + # Auto-improve: confirmed production failures -> improvement report -> code fix (via + # the AgentX-Eval-Skill auto-improve skill). Self-host only. + self.improvement_groups = ImprovementGroupsClient(api_key=api_key, base_url=self._api_root()) self.profile = MonitorProfileClient(self) # Legacy view of an LLM Judge Scorer's online profile - constructed lazily so its # DeprecationWarning fires on first USE, not for every client that never touches it. @@ -317,8 +338,12 @@ def calibration(self, window: str = "7d") -> "CalibrationSummary": AgentX's own verdicts agreed with real-world ground truth reported later (ops outcomes via ``client.outcomes``, end-user downvotes, and human review labels). Returns the dashboard's Judge Calibration numbers with these exact keys: ``comparedCount``, - ``agreementRate``, ``falsePositiveRate``, ``falseNegativeRate`` (plus - ``reportedCount``/``reviewLabelCount``/``noVerdictCount``). Per-scorer calibration + ``agreementRate``, ``falsePositiveRate``, ``falseNegativeRate``, ``alpha``, + ``alphaBand``, ``alphaMinItems`` (plus + ``reportedCount``/``reviewLabelCount``/``noVerdictCount``). ``agreementRate`` is raw + agreement and inflates under class imbalance; ``alpha`` is the chance-corrected + version (Krippendorff's alpha - null until ``alphaMinItems`` labeled pairs exist). + Per-scorer calibration lives on ``client.monitor.judge_scorers.calibration(scorer_id)``.""" return CalibrationSummary( self._request( diff --git a/agentx/monitor/improvement_groups.py b/agentx/monitor/improvement_groups.py new file mode 100644 index 0000000..e1c87fb --- /dev/null +++ b/agentx/monitor/improvement_groups.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +import requests + +from agentx.util import api_base, get_headers + + +class AgentXImprovementGroupsError(Exception): + pass + + +class ImprovementGroupsClient: + """Surfaced as ``client.monitor.improvement_groups``: the auto-improve loop's accumulator. + + Batch lifecycle: one COLLECTING group at a time. Every Confirm verdict in signal review + automatically lands the confirmed failure there - accumulation is free, declining is + choosing Ignore. ``generate_report`` SPENDS the batch: one LLM pass clusters the confirmed + failures into issues with recommendations, the group is sealed onto that report (keeping + exactly its source cases), and the pending accumulator is thereby cleared - the next + Confirm starts a fresh batch, and the next generate makes a new report from it. The report's id is the + hand-off: paste it into the AgentX-Eval-Skill ``auto-improve`` skill, which fetches the + report (``get_report``) and triages the fixes against your agent's actual source code. + + Evidence here is exclusively ONLINE - production verdicts a human confirmed - never + offline dataset runs. Self-host only. + """ + + def __init__(self, api_key: Optional[str] = None, base_url: Optional[str] = None): + self._api_key = api_key + self._base_url = (base_url or api_base()).rstrip("/") + + def _request(self, method: str, path: str, json: Any = None, timeout: int = 120) -> Any: + resp = requests.request( + method, + f"{self._base_url}/agent-monitoring{path}", + headers={**get_headers(self._api_key), "Content-Type": "application/json"}, + json=json, + timeout=timeout, + ) + if resp.status_code >= 400: + try: + detail = resp.json().get("error", resp.reason) + except ValueError: + detail = resp.reason + raise AgentXImprovementGroupsError(f"Improvement group request failed ({resp.status_code}): {detail}") + return resp.json() if resp.text else {} + + def list(self) -> List[Dict[str, Any]]: + return self._request("GET", "/improvement-groups").get("improvementGroups", []) + + def get(self, group_id: str) -> Dict[str, Any]: + """The group with its members - each a confirmed failure's evidence snapshot.""" + return self._request("GET", f"/improvement-groups/{group_id}")["improvementGroup"] + + def remove_member(self, group_id: str, member_id: str) -> None: + """Prune a member before spending the group (a confirm that turned out uninteresting).""" + self._request("DELETE", f"/improvement-groups/{group_id}/members/{member_id}") + + def generate_report(self, group_id: str, model: Optional[str] = None) -> Dict[str, Any]: + """Spend the group: one real LLM call clustering the confirmed failures into issues + with recommendations. Returns the report; its ``_id`` is what the auto-improve skill + takes. Explicit and billed - never called implicitly.""" + payload: Dict[str, Any] = {} + if model is not None: + payload["model"] = model + return self._request("POST", f"/improvement-groups/{group_id}/report", json=payload, timeout=300)["report"] + + def list_reports(self) -> List[Dict[str, Any]]: + return self._request("GET", "/improvement-reports").get("improvementReports", []) + + def get_report(self, report_id: str) -> Dict[str, Any]: + """Fetch a report by the id the dashboard (or generate_report) handed out - the exact + call the auto-improve skill makes.""" + return self._request("GET", f"/improvement-reports/{report_id}")["report"] diff --git a/agentx/monitor/judge_scorers.py b/agentx/monitor/judge_scorers.py index 00ed507..f479f25 100644 --- a/agentx/monitor/judge_scorers.py +++ b/agentx/monitor/judge_scorers.py @@ -258,11 +258,19 @@ def _profile_id(self, scorer_id: str) -> str: def calibration(self, scorer_id: str, window: str = "7d") -> dict: """How this scorer's verdicts compare against recorded ground truth (triage - corrections, outcomes, end-user votes) over the window.""" + corrections, outcomes, end-user votes) over the window. Beyond the raw + ``agreementRate``, the response carries ``alpha``/``alphaBand`` (chance-corrected + agreement - Krippendorff's alpha, null until ``alphaMinItems`` labeled pairs exist) + and ``ratingMae`` (mean absolute error against human re-scores, over the + ``withCorrectedScore`` pairs that carry a number). ``window`` accepts "24h", "7d", + "30d", or "rubric" - only verdicts produced by the CURRENT rubric (since its criteria + were last edited, clamped to 30 days), which is what the dashboard's Tune Judge flow + uses by default; the response's ``window``/``since`` echo the boundary applied.""" return self._request("GET", f"/online-evaluators/{self._profile_id(scorer_id)}/calibration?window={window}") def tune(self, scorer_id: str, window: str = "7d") -> dict: - """Propose a rewrite of the rubric from calibration disagreements (LLM call, slow).""" + """Propose a rewrite of the rubric from calibration disagreements (LLM call, slow). + ``window`` accepts the same values as :meth:`calibration`, including "rubric".""" data = self._request( "POST", f"/online-evaluators/{self._profile_id(scorer_id)}/tune", json={"window": window}, timeout=300 )