diff --git a/agentx/monitor/client.py b/agentx/monitor/client.py index 16acf29..96a4ffa 100644 --- a/agentx/monitor/client.py +++ b/agentx/monitor/client.py @@ -187,10 +187,17 @@ def _api_root(self) -> str: return self._base_url[: -len(suffix)] return self._base_url - def _request(self, method: str, path: str, timeout: int = 30, base: Optional[str] = None, **kwargs) -> Any: + def _request( + self, method: str, path: str, timeout: int = 30, base: Optional[str] = None, retry: bool = True, **kwargs + ) -> Any: + # retry=False for non-idempotent judge-spending POSTs (sweep, coherence, portability, + # tuning): a client-side timeout must not fire the same LLM-billing work a second time + # while the first invocation is still running server-side. Same precedent as + # EvaluationsClient._request / analyze_run. url = f"{base or self._base_url}{path}" last_exc: Optional[Exception] = None - for attempt, wait in enumerate([0.0] + _RETRY_BACKOFF): + schedule = [0.0] + _RETRY_BACKOFF if retry else [0.0] + for attempt, wait in enumerate(schedule): if wait: time.sleep(wait) try: @@ -204,7 +211,7 @@ def _request(self, method: str, path: str, timeout: int = 30, base: Optional[str raise AgentXAuthError("Invalid or missing API key") if resp.status_code == 422: raise AgentXValidationError(resp.text) - if resp.status_code in _RETRYABLE_STATUS and attempt < _MAX_RETRIES - 1: + if retry and resp.status_code in _RETRYABLE_STATUS and attempt < _MAX_RETRIES - 1: logger.debug( "Retryable status %d (attempt %d)", resp.status_code, attempt + 1 ) @@ -421,7 +428,7 @@ def run_session_coherence_check(self, session_id: str) -> dict: button. Raises AgentXMonitorError if the engine has no judge key configured.""" data = self._request( "POST", f"/agent-monitoring/sessions/{session_id}/coherence-check", - base=self._api_root(), timeout=180, + base=self._api_root(), timeout=180, retry=False, ) return data.get("score", data) if isinstance(data, dict) else data @@ -444,7 +451,7 @@ def run_session_sweep(self) -> dict: engines run this automatically every minute; the manual trigger exists for demos, tests, and backfills. Returns ``{"judged": n}``.""" return self._request( - "POST", "/agent-monitoring/session-sweep/run", base=self._api_root(), timeout=300 + "POST", "/agent-monitoring/session-sweep/run", base=self._api_root(), timeout=300, retry=False ) # ------------------------------------------------------------------ @@ -457,7 +464,7 @@ def run_model_portability(self, trace_id: str, model_ids: List[str]) -> dict: plus judging, so expect tens of seconds.""" return self._request( "POST", f"/agent-monitoring/traces/{trace_id}/portability", - base=self._api_root(), json={"modelIds": model_ids}, timeout=300, + base=self._api_root(), json={"modelIds": model_ids}, timeout=300, retry=False, ) # ------------------------------------------------------------------ diff --git a/agentx/monitor/patterns.py b/agentx/monitor/patterns.py index dce2736..ee292ac 100644 --- a/agentx/monitor/patterns.py +++ b/agentx/monitor/patterns.py @@ -114,6 +114,10 @@ def builder( agent_ids=agent_ids, ) + def delete(self, pattern_id: str) -> None: + """Delete a pattern. Its historical signals remain as history.""" + self._client._request("DELETE", f"/agent-monitoring/patterns/{pattern_id}", base=self._client._api_root()) + def get(self, pattern_id: str) -> MonitorPattern: return self._client.get_pattern(pattern_id) diff --git a/agentx/monitor/scorer_groups.py b/agentx/monitor/scorer_groups.py index 69f22e1..a712455 100644 --- a/agentx/monitor/scorer_groups.py +++ b/agentx/monitor/scorer_groups.py @@ -67,8 +67,8 @@ def create( """``members``: [{"kind": "judge"|"pattern"|"custom", "refId": ..., "weight": 1, "gate": False}]. ``online``: {"enabled": True, "sampleRate": 0.1, "alertThreshold": 5, "severity": "medium"}. Add ``"scope": "session", "idleSeconds": 120`` to score whole multi-turn sessions once - idle, instead of each sampled trace. - or None for offline-only.""" + idle, instead of each sampled trace. Pass ``online=None`` (the default) for a group + that only grades offline dataset runs.""" payload: Dict[str, Any] = {"name": name, "members": members} if description is not None: payload["description"] = description diff --git a/agentx/monitor/sessions.py b/agentx/monitor/sessions.py index e633451..562c0b4 100644 --- a/agentx/monitor/sessions.py +++ b/agentx/monitor/sessions.py @@ -22,8 +22,9 @@ def spans(self, session_id: str) -> List[dict]: def scores(self, session_id: str) -> List[dict]: """Session-level verdicts, newest first. ``kind`` says who scored: a session-scoped - online evaluator (``online-eval:``) or a session-scoped scorer group - (``scorer-group:``).""" + online evaluator (``online-eval:``), a session-scoped scorer group + (``scorer-group:``), or legacy ``"coherence"`` rows written before the Session + Baseline Judge existed - branch defensively on unknown kinds.""" return self._client.list_session_scores(session_id) def run_sweep(self) -> dict: diff --git a/tests/test_selfhost_analysis_fallback.py b/tests/test_selfhost_analysis_fallback.py index fa48c36..85d8f3e 100644 --- a/tests/test_selfhost_analysis_fallback.py +++ b/tests/test_selfhost_analysis_fallback.py @@ -217,6 +217,12 @@ def test_the_fallback_request_gets_the_long_analysis_timeout(): assert kwargs["timeout"] > 60, "a synchronous judge pass needs more than the 30s default" assert kwargs["json"]["judges"] == [{"model": "gpt-5.6-luna"}] + # judges=None must OMIT the key - the engine then scores with its platform default model; + # injecting a hosted-only default (the old "gpt-5.5") produced uncallable judges. + client.analyze_run(RUN) + _, _, kwargs = session.calls[-1] + assert "judges" not in kwargs["json"] + # --------------------------------------------------------------------------- # Only a 404 means "wrong engine"