auto improve from online eval with skill

agentx/monitor/client.py

@@ -67,6 +67,22 @@ def reported_count(self) -> int:
     def review_label_count(self) -> int:
         return int(self.get("reviewLabelCount") or 0)
 
+    @property
+    def alpha(self):
+        """Chance-corrected agreement (Krippendorff's alpha over the binary verdict pair) -
+        the raw ``agreement_rate`` corrected for what a weighted coin would score on this
+        label mix. ``None`` below the server's sample floor (``alphaMinItems`` labeled pairs)
+        or when every label is identical: withheld, never fabricated. 1 = perfect, 0 = no
+        better than chance, negative = systematically opposed."""
+        return self.get("alpha")
+
+    @property
+    def alpha_band(self):
+        """Human-readable band for ``alpha`` (poor/slight/fair/moderate/substantial/
+        near-perfect), computed server-side so every surface reads the same alpha the
+        same way."""
+        return self.get("alphaBand")
+
 
 class MonitorClient:
     """Low-level HTTP client for the Monitor API (``/monitor``). Accessed via
@@ -131,6 +147,11 @@ def __init__(
         # surface that matches the product; evaluations.settings and online_evaluators below
         # remain as its profile-level views.
         self.judge_scorers = JudgeScorersClient(api_key=api_key, base_url=self._api_root())
+        from agentx.monitor.improvement_groups import ImprovementGroupsClient
+
+        # Auto-improve: confirmed production failures -> improvement report -> code fix (via
+        # the AgentX-Eval-Skill auto-improve skill). Self-host only.
+        self.improvement_groups = ImprovementGroupsClient(api_key=api_key, base_url=self._api_root())
         self.profile = MonitorProfileClient(self)
         # Legacy view of an LLM Judge Scorer's online profile - constructed lazily so its
         # DeprecationWarning fires on first USE, not for every client that never touches it.
@@ -317,8 +338,12 @@ def calibration(self, window: str = "7d") -> "CalibrationSummary":
         AgentX's own verdicts agreed with real-world ground truth reported later (ops outcomes
         via ``client.outcomes``, end-user downvotes, and human review labels). Returns the
         dashboard's Judge Calibration numbers with these exact keys: ``comparedCount``,
-        ``agreementRate``, ``falsePositiveRate``, ``falseNegativeRate`` (plus
-        ``reportedCount``/``reviewLabelCount``/``noVerdictCount``). Per-scorer calibration
+        ``agreementRate``, ``falsePositiveRate``, ``falseNegativeRate``, ``alpha``,
+        ``alphaBand``, ``alphaMinItems`` (plus
+        ``reportedCount``/``reviewLabelCount``/``noVerdictCount``). ``agreementRate`` is raw
+        agreement and inflates under class imbalance; ``alpha`` is the chance-corrected
+        version (Krippendorff's alpha - null until ``alphaMinItems`` labeled pairs exist).
+        Per-scorer calibration
         lives on ``client.monitor.judge_scorers.calibration(scorer_id)``."""
         return CalibrationSummary(
             self._request(

agentx/monitor/improvement_groups.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+import requests
+
+from agentx.util import api_base, get_headers
+
+
+class AgentXImprovementGroupsError(Exception):
+    pass
+
+
+class ImprovementGroupsClient:
+    """Surfaced as ``client.monitor.improvement_groups``: the auto-improve loop's accumulator.
+
+    Batch lifecycle: one COLLECTING group at a time. Every Confirm verdict in signal review
+    automatically lands the confirmed failure there - accumulation is free, declining is
+    choosing Ignore. ``generate_report`` SPENDS the batch: one LLM pass clusters the confirmed
+    failures into issues with recommendations, the group is sealed onto that report (keeping
+    exactly its source cases), and the pending accumulator is thereby cleared - the next
+    Confirm starts a fresh batch, and the next generate makes a new report from it. The report's id is the
+    hand-off: paste it into the AgentX-Eval-Skill ``auto-improve`` skill, which fetches the
+    report (``get_report``) and triages the fixes against your agent's actual source code.
+
+    Evidence here is exclusively ONLINE - production verdicts a human confirmed - never
+    offline dataset runs. Self-host only.
+    """
+
+    def __init__(self, api_key: Optional[str] = None, base_url: Optional[str] = None):
+        self._api_key = api_key
+        self._base_url = (base_url or api_base()).rstrip("/")
+
+    def _request(self, method: str, path: str, json: Any = None, timeout: int = 120) -> Any:
+        resp = requests.request(
+            method,
+            f"{self._base_url}/agent-monitoring{path}",
+            headers={**get_headers(self._api_key), "Content-Type": "application/json"},
+            json=json,
+            timeout=timeout,
+        )
+        if resp.status_code >= 400:
+            try:
+                detail = resp.json().get("error", resp.reason)
+            except ValueError:
+                detail = resp.reason
+            raise AgentXImprovementGroupsError(f"Improvement group request failed ({resp.status_code}): {detail}")
+        return resp.json() if resp.text else {}
+
+    def list(self) -> List[Dict[str, Any]]:
+        return self._request("GET", "/improvement-groups").get("improvementGroups", [])
+
+    def get(self, group_id: str) -> Dict[str, Any]:
+        """The group with its members - each a confirmed failure's evidence snapshot."""
+        return self._request("GET", f"/improvement-groups/{group_id}")["improvementGroup"]
+
+    def remove_member(self, group_id: str, member_id: str) -> None:
+        """Prune a member before spending the group (a confirm that turned out uninteresting)."""
+        self._request("DELETE", f"/improvement-groups/{group_id}/members/{member_id}")
+
+    def generate_report(self, group_id: str, model: Optional[str] = None) -> Dict[str, Any]:
+        """Spend the group: one real LLM call clustering the confirmed failures into issues
+        with recommendations. Returns the report; its ``_id`` is what the auto-improve skill
+        takes. Explicit and billed - never called implicitly."""
+        payload: Dict[str, Any] = {}
+        if model is not None:
+            payload["model"] = model
+        return self._request("POST", f"/improvement-groups/{group_id}/report", json=payload, timeout=300)["report"]
+
+    def list_reports(self) -> List[Dict[str, Any]]:
+        return self._request("GET", "/improvement-reports").get("improvementReports", [])
+
+    def get_report(self, report_id: str) -> Dict[str, Any]:
+        """Fetch a report by the id the dashboard (or generate_report) handed out - the exact
+        call the auto-improve skill makes."""
+        return self._request("GET", f"/improvement-reports/{report_id}")["report"]

agentx/monitor/judge_scorers.py

@@ -258,11 +258,19 @@ def _profile_id(self, scorer_id: str) -> str:
 
     def calibration(self, scorer_id: str, window: str = "7d") -> dict:
         """How this scorer's verdicts compare against recorded ground truth (triage
-        corrections, outcomes, end-user votes) over the window."""
+        corrections, outcomes, end-user votes) over the window. Beyond the raw
+        ``agreementRate``, the response carries ``alpha``/``alphaBand`` (chance-corrected
+        agreement - Krippendorff's alpha, null until ``alphaMinItems`` labeled pairs exist)
+        and ``ratingMae`` (mean absolute error against human re-scores, over the
+        ``withCorrectedScore`` pairs that carry a number). ``window`` accepts "24h", "7d",
+        "30d", or "rubric" - only verdicts produced by the CURRENT rubric (since its criteria
+        were last edited, clamped to 30 days), which is what the dashboard's Tune Judge flow
+        uses by default; the response's ``window``/``since`` echo the boundary applied."""
         return self._request("GET", f"/online-evaluators/{self._profile_id(scorer_id)}/calibration?window={window}")
 
     def tune(self, scorer_id: str, window: str = "7d") -> dict:
-        """Propose a rewrite of the rubric from calibration disagreements (LLM call, slow)."""
+        """Propose a rewrite of the rubric from calibration disagreements (LLM call, slow).
+        ``window`` accepts the same values as :meth:`calibration`, including "rubric"."""
         data = self._request(
             "POST", f"/online-evaluators/{self._profile_id(scorer_id)}/tune", json={"window": window}, timeout=300
         )