From 1b60b67b741b066760d58ee11c71b3f71c3bda53 Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:38:50 +0000 Subject: [PATCH 1/6] feat(collector): stamp harness provenance on source and SDK events --- crates/fpai-collect/src/sources/hooks/mod.rs | 11 ++- .../src/sources/hooks/transform.rs | 4 +- crates/fpai-collect/src/spool.rs | 97 +++++++++++++++++++ crates/fpai-collect/src/uploader.rs | 16 ++- crates/fpai-collect/tests/hooks_source.rs | 54 +++++++++++ crates/fpai-collect/tests/uploader.rs | 42 ++++++++ 6 files changed, 217 insertions(+), 7 deletions(-) diff --git a/crates/fpai-collect/src/sources/hooks/mod.rs b/crates/fpai-collect/src/sources/hooks/mod.rs index 53446f28c..be6c8abe0 100644 --- a/crates/fpai-collect/src/sources/hooks/mod.rs +++ b/crates/fpai-collect/src/sources/hooks/mod.rs @@ -256,6 +256,7 @@ async fn poll_once( .or_insert_with(|| AllowBucket { session_id: key.0.clone(), agent_id: transform::agent_id(&row), + integration: key.5.clone(), event_name: key.1.clone(), tool_name: key.2.clone(), minute_ms: key.3, @@ -273,7 +274,10 @@ async fn poll_once( } for event in transform::to_events(&row, this_offset, environment) { - writer.push(event).await.map_err(io_err)?; + writer + .push_with_harness(event, row.integration.as_deref().unwrap_or("hooks")) + .await + .map_err(io_err)?; emitted += 1; } } @@ -293,7 +297,10 @@ async fn poll_once( for bucket in buckets.values() { if let Some(event) = bucket.to_event(environment) { - writer.push(event).await.map_err(io_err)?; + writer + .push_with_harness(event, &bucket.integration) + .await + .map_err(io_err)?; emitted += 1; } } diff --git a/crates/fpai-collect/src/sources/hooks/transform.rs b/crates/fpai-collect/src/sources/hooks/transform.rs index e3eb312a7..a7e2da507 100644 --- a/crates/fpai-collect/src/sources/hooks/transform.rs +++ b/crates/fpai-collect/src/sources/hooks/transform.rs @@ -826,6 +826,7 @@ pub fn to_events(row: &HookRow, offset: u64, environment: &str) -> Vec { pub struct AllowBucket { pub session_id: String, pub agent_id: String, + pub integration: String, pub event_name: String, pub tool_name: Option, /// Truncated to the minute; also the bucket's emitted timestamp. @@ -852,7 +853,7 @@ pub struct AllowBucket { /// emitted is wrong for the rest, which is worse than no attribution: it moves /// a count into a bucket someone is using to judge a rollout. Splitting instead /// costs extra events only when a minute genuinely mixed sources. -pub type BucketKey = (String, String, Option, i64, Attribution); +pub type BucketKey = (String, String, Option, i64, Attribution, String); pub fn bucket_key(row: &HookRow) -> Option { let session = row.session_id.clone().filter(|s| !s.is_empty())?; @@ -864,6 +865,7 @@ pub fn bucket_key(row: &HookRow) -> Option { row.tool_name.clone(), minute, Attribution::of(row), + row.integration.clone().unwrap_or_else(|| "hooks".into()), )) } diff --git a/crates/fpai-collect/src/spool.rs b/crates/fpai-collect/src/spool.rs index 562774ace..c1173ce17 100644 --- a/crates/fpai-collect/src/spool.rs +++ b/crates/fpai-collect/src/spool.rs @@ -44,6 +44,18 @@ const MIN_BATCH_BYTES: u64 = 64 * 1024; /// Default batch cap, matching the ingest endpoint's body limit. pub const DEFAULT_MAX_BATCH_BYTES: u64 = 8 * 1024 * 1024; +/// Fill provenance only when absent. SDK batches arrive already spooled, so +/// the uploader uses this same stamp without writing a second batch. +pub(crate) fn stamp_harness(event: &mut Value, harness: &str) -> bool { + if let Some(obj) = event.as_object_mut() + && !obj.contains_key("harness") + { + obj.insert("harness".into(), Value::String(harness.into())); + return true; + } + false +} + /// Accumulates events and writes them out as size-bounded batches. pub struct SpoolWriter { dir: PathBuf, @@ -171,6 +183,8 @@ impl SpoolWriter { // disk. self.redacted += crate::redact::scrub_value(&mut event, self.redact) as u64; + stamp_harness(&mut event, &self.prefix); + // Stamp the machine id here, at the one point every event passes // through — a source cannot forget it and a new source inherits it. Set // only when absent, so an event that already names its machine (a @@ -245,6 +259,16 @@ impl SpoolWriter { Ok(()) } + /// Hook batches contain multiple CLIs. Use the row's CLI, not its agent id. + pub async fn push_with_harness( + &mut self, + mut event: Value, + harness: &str, + ) -> std::io::Result<()> { + stamp_harness(&mut event, harness); + self.push(event).await + } + /// Durably write the buffered events as one batch. No-op when empty. /// /// Callers MUST await this before advancing any persisted cursor. The @@ -363,6 +387,79 @@ mod tests { v } + #[tokio::test] + async fn every_source_stamps_harness_without_guessing_from_the_agent_id() { + for source in [ + "claude", + "codex", + "copilot", + "cursor", + "pi", + "factory", + "antigravity", + "goose", + "opencode", + "hermes", + "devin", + "openclaw", + "sdk", + ] { + let dir = tmpdir(source); + let mut w = SpoolWriter::new(dir.clone(), DEFAULT_MAX_BATCH_BYTES, source, "s") + .with_label(Some("extra-path".into())); + for kind in ["agent_start", "tool_use", "agent_end"] { + w.push(json!({"type": kind, "agent_id": "not-a-harness"})) + .await + .unwrap(); + } + w.flush().await.unwrap(); + let body = std::fs::read_to_string(dir.join(batches(&dir).remove(0))).unwrap(); + for line in body.lines() { + let event: Value = serde_json::from_str(line).unwrap(); + assert_eq!(event["harness"], source); + } + std::fs::remove_dir_all(&dir).ok(); + } + } + + #[tokio::test] + async fn hooks_use_the_rows_cli_and_existing_harness_is_untouched() { + let dir = tmpdir("harness-hooks"); + let mut w = SpoolWriter::new(dir.clone(), DEFAULT_MAX_BATCH_BYTES, "hooks", "s"); + for cli in ["claude", "codex", "hermes"] { + w.push_with_harness(json!({"type": "hook_completed"}), cli) + .await + .unwrap(); + } + for harness in [json!("original"), Value::Null, json!("")] { + w.push(json!({"harness": harness})).await.unwrap(); + w.push_with_harness(json!({"harness": harness}), "copilot") + .await + .unwrap(); + } + w.flush().await.unwrap(); + let body = std::fs::read_to_string(dir.join(batches(&dir).remove(0))).unwrap(); + let values: Vec = body + .lines() + .map(|line| serde_json::from_str::(line).unwrap()["harness"].clone()) + .collect(); + assert_eq!( + values, + vec![ + json!("claude"), + json!("codex"), + json!("hermes"), + json!("original"), + json!("original"), + Value::Null, + Value::Null, + json!(""), + json!("") + ] + ); + std::fs::remove_dir_all(&dir).ok(); + } + #[tokio::test] async fn writes_one_batch_and_leaves_no_tmp_behind() { let dir = tmpdir("basic"); diff --git a/crates/fpai-collect/src/uploader.rs b/crates/fpai-collect/src/uploader.rs index ee527d9a8..dccb94536 100644 --- a/crates/fpai-collect/src/uploader.rs +++ b/crates/fpai-collect/src/uploader.rs @@ -268,7 +268,14 @@ impl Uploader { Err(e) => return Err(UploadError::Io(e)), }; - let bytes = redact_batch(&bytes, self.redact); + // Both SDKs (including the legacy spool) publish `event-*.jsonl`. + // The name survives parking/retries, so provenance is stable even + // after the batch moves out of custom-agents/events into failed/. + let sdk = path + .file_name() + .and_then(|s| s.to_str()) + .is_some_and(|name| name.starts_with("event-")); + let bytes = prepare_batch(&bytes, self.redact, sdk); for chunk in split_lines(&bytes, self.max_upload_bytes) { self.post_batch(path, chunk).await?; } @@ -698,8 +705,8 @@ fn lower_hex(bytes: &[u8]) -> String { out } -fn redact_batch(bytes: &[u8], mode: Redact) -> Vec { - if mode == Redact::Off { +fn prepare_batch(bytes: &[u8], mode: Redact, sdk: bool) -> Vec { + if mode == Redact::Off && !sdk { return bytes.to_vec(); } @@ -710,7 +717,8 @@ fn redact_batch(bytes: &[u8], mode: Redact) -> Vec { .map_or((line, false), |body| (body, true)); match serde_json::from_slice::(body) { Ok(mut event) => { - if crate::redact::scrub_value(&mut event, mode) > 0 { + let stamped = sdk && crate::spool::stamp_harness(&mut event, "sdk"); + if crate::redact::scrub_value(&mut event, mode) > 0 || stamped { event .serialize(&mut serde_json::Serializer::new(&mut out)) .expect("serializing JSON into Vec cannot fail"); diff --git a/crates/fpai-collect/tests/hooks_source.rs b/crates/fpai-collect/tests/hooks_source.rs index 5c94ab06e..8e939aca2 100644 --- a/crates/fpai-collect/tests/hooks_source.rs +++ b/crates/fpai-collect/tests/hooks_source.rs @@ -115,6 +115,60 @@ fn epoch_millis_become_rfc3339_with_microseconds() { assert!(s < s1); } +#[tokio::test] +async fn hooks_stamp_each_cli_on_pairs_and_separate_allow_buckets() { + for verbosity in [HooksVerbosity::All, HooksVerbosity::Decisions] { + let store = tmpdir("harness-store"); + let state = tmpdir("harness-state"); + let spool = tmpdir("harness-spool"); + let mut rows = String::new(); + for cli in [ + "claude", + "codex", + "copilot", + "cursor", + "pi", + "factory", + "antigravity", + "goose", + "opencode", + "hermes", + "devin", + "openclaw", + ] { + for decision in ["allow", "deny"] { + let mut row: Value = + serde_json::from_str(&allow_row(1785740912184, "Bash")).unwrap(); + row["integration"] = serde_json::json!(cli); + row["decision"] = serde_json::json!(decision); + rows.push_str(&row.to_string()); + rows.push('\n'); + } + } + fs::write(store.join("page.jsonl"), rows).unwrap(); + run_once(&store, &state, &spool, verbosity).await; + let events = spooled(&spool); + assert_eq!( + events.len(), + if verbosity == HooksVerbosity::All { + 48 + } else { + 36 + } + ); + for event in events { + let cli = event["harness"].as_str().unwrap(); + assert_eq!(event["agent_id"], format!("{cli}-repo")); + if verbosity == HooksVerbosity::Decisions && event["outcome"] == "allow" { + assert_eq!(event["failproofai_allow_count"], 1); + } + } + fs::remove_dir_all(&store).ok(); + fs::remove_dir_all(&state).ok(); + fs::remove_dir_all(&spool).ok(); + } +} + #[test] fn a_leap_day_is_handled() { // The reason this uses a date library rather than hand-rolled maths. diff --git a/crates/fpai-collect/tests/uploader.rs b/crates/fpai-collect/tests/uploader.rs index 68cb1cb6c..7a7f5730b 100644 --- a/crates/fpai-collect/tests/uploader.rs +++ b/crates/fpai-collect/tests/uploader.rs @@ -88,6 +88,48 @@ async fn a_2xx_with_an_accepting_ack_deletes_the_batch() { fs::remove_dir_all(&failed).ok(); } +#[tokio::test] +async fn sdk_harness_is_stamped_even_without_redaction_and_survives_retry() { + for mode in [Redact::Minimal, Redact::Off] { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "accepted": 3, "skipped": 0 + }))) + .mount(&server) + .await; + let spool = tmpdir("sdk-harness"); + let failed = tmpdir("sdk-harness-failed"); + // Parked SDK files retain their event-* stem. + let batch = spool.join("event-s-1-0.a1.jsonl"); + fs::write( + &batch, + concat!( + "{\"type\":\"agent_start\",\"agent_id\":\"claude-custom\"}\n", + "{\"type\":\"tool_use\",\"harness\":\"original\"}\n", + "{\"type\":\"agent_end\",\"harness\":null}\n", + ), + ) + .unwrap(); + uploader(&server, &failed) + .with_redact(mode) + .upload_file(&batch) + .await + .unwrap(); + let requests = server.received_requests().await.unwrap(); + let body = String::from_utf8(requests[0].body.clone()).unwrap(); + let events: Vec = body + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + assert_eq!(events[0]["harness"], "sdk"); + assert_eq!(events[1]["harness"], "original"); + assert!(events[2]["harness"].is_null()); + fs::remove_dir_all(&spool).ok(); + fs::remove_dir_all(&failed).ok(); + } +} + #[tokio::test] async fn sdk_batches_are_redacted_before_upload() { let server = MockServer::start().await; From 84e2e5c040eae73227c8770e5fa5bcbb0b5d484f Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:53:48 +0000 Subject: [PATCH 2/6] feat(fp): add issue filters, facets and older-Cloud warnings --- fp-cloud-cli/fp_cli/client.py | 63 +++++- .../fp_cli/commands/incidents_cmds.py | 120 +++++++++- fp-cloud-cli/fp_cli/models.py | 3 + fp-cloud-cli/fp_cli/output.py | 31 ++- fp-cloud-cli/tests/test_issue_filters.py | 206 ++++++++++++++++++ 5 files changed, 409 insertions(+), 14 deletions(-) create mode 100644 fp-cloud-cli/tests/test_issue_filters.py diff --git a/fp-cloud-cli/fp_cli/client.py b/fp-cloud-cli/fp_cli/client.py index d60c855e6..4716a46de 100644 --- a/fp-cloud-cli/fp_cli/client.py +++ b/fp-cloud-cli/fp_cli/client.py @@ -672,6 +672,7 @@ def _cloud_not_found(exc: FpCliError, path: str, *, body=None, params=None) -> N or family == "queries" and segments[1:] in (["run"], ["schema"]) or family == "audits" and segments == ["audits", "contracts"] or family == "alerts" and segments == ["alerts", "recipients"] + or family == "issues" and segments in (["issues", "count"], ["issues", "facets"]) or family == "users" and segments == ["users", "defaults"] or family == "usage" and segments == ["usage", "windows"] or family == "settings" and segments == ["settings", "model-context-windows"] @@ -1300,20 +1301,32 @@ def list_incidents( state: Optional[str] = None, alert_id: Optional[str] = None, limit: Optional[int] = None, + filters: Optional[Dict[str, str]] = None, + on_filters_unavailable: Optional[Callable[[], None]] = None, ) -> List[Incident]: """GET /api/issues (or /api/alerts/{id}/issues when alert_id given). The alert-scoped path honours ``state``/``limit`` too — pass them so the filters aren't silently dropped on that path (``_get_json`` omits None params).""" - if alert_id: + if alert_id and not filters: data = _get_json(ctx, f"/api/alerts/{alert_id}/issues", {"state": state, "limit": limit}) else: - data = _get_json(ctx, "/api/issues", {"state": state, "limit": limit}) - return [Incident.from_dict(i) for i in (data if isinstance(data, list) else [])] - - -def count_incidents(ctx: ClientContext, *, state: Optional[str] = None) -> int: + params = {"state": state, "limit": limit, **(filters or {})} + if alert_id: + params["alert_id"] = alert_id + data = cloud_request(ctx, "GET", "/api/issues", params=params) + _check_issue_filters(ctx, data, filters, on_filters_unavailable) + items = data.get("issues", []) if isinstance(data, dict) else data + return [Incident.from_dict(i) for i in (items if isinstance(items, list) else [])] + + +def count_incidents( + ctx: ClientContext, *, state: Optional[str] = None, + filters: Optional[Dict[str, str]] = None, + on_filters_unavailable: Optional[Callable[[], None]] = None, +) -> int: """GET /api/issues/count — {count}.""" - data = _get_json(ctx, "/api/issues/count", {"state": state}) + data = cloud_request(ctx, "GET", "/api/issues/count", params={"state": state, **(filters or {})}) + _check_issue_filters(ctx, data, filters, on_filters_unavailable) if not isinstance(data, dict): return 0 try: @@ -1322,6 +1335,42 @@ def count_incidents(ctx: ClientContext, *, state: Optional[str] = None) -> int: return 0 +def issue_facets( + ctx: ClientContext, *, state: Optional[str] = None, + filters: Optional[Dict[str, str]] = None, +) -> Dict[str, Any]: + """GET /api/issues/facets — available scope/source/assignee values with counts.""" + data = cloud_request(ctx, "GET", "/api/issues/facets", params={"state": state, **(filters or {})}) + if not isinstance(data, dict): + raise ApiError("Cloud returned an invalid issues facets response") + return data + + +def _check_issue_filters( + ctx: ClientContext, data: Any, filters: Optional[Dict[str, str]], + on_unavailable: Optional[Callable[[], None]], +) -> None: + """An echo proves support; otherwise probe facets, never infer it from rows. + + Older deployments return a bare, unfiltered array/count. Only a missing + capability route warrants the compatibility warning; auth/network/server + failures must remain visible, not be misreported as an older Cloud. + """ + if not filters: + return + echoed = data.get("filters", data.get("applied_filters")) if isinstance(data, dict) else None + if isinstance(echoed, dict) and all( + echoed.get(key) == value or echoed.get(key) == value.split(",") + for key, value in filters.items() + ): + return + try: + issue_facets(ctx, filters=filters) + except NotFoundError: + if on_unavailable: + on_unavailable() + + def get_incident(ctx: ClientContext, incident_id: str) -> Incident: """GET /api/issues/{id} — full detail (comments, subscribers, activity).""" return Incident.from_dict(_get_json(ctx, f"/api/issues/{incident_id}")) diff --git a/fp-cloud-cli/fp_cli/commands/incidents_cmds.py b/fp-cloud-cli/fp_cli/commands/incidents_cmds.py index 741c996bd..3ade01db6 100644 --- a/fp-cloud-cli/fp_cli/commands/incidents_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/incidents_cmds.py @@ -13,8 +13,8 @@ import typer from .. import client as api -from .. import output -from .._context import GLOBALS_EPILOG, AppState, require_auth, validate_limit +from .. import dates, output +from .._context import GLOBALS_EPILOG, AppState, require_auth, resolve_dates, validate_limit from ..errors import ApiError, ForbiddenError, NotFoundError from . import _write @@ -23,6 +23,7 @@ # TERMINAL state (won't-fix); missing it here would reject `--state closed` # client-side with exit 2 for a state the server accepts. _STATES = ("firing", "acknowledged", "resolved", "closed") +_SOURCES = ("alert", "audit", "manual") _UUID_RE = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$") @@ -68,12 +69,71 @@ def _fail(state: AppState, exc: Exception, *, incident_id: str = "") -> None: raise exc +def _issue_filters( + agent, harness, environment, machine, source, alert, audit, assignee, + since, until, search, +) -> dict: + """The shared list/count/facets contract; repeated values become CSV.""" + filters = {} + for key, values in ( + ("agent", agent), ("harness", harness), ("environment", environment), + ("machine", machine), ("source", source), ("alert_id", alert), + ("audit_id", audit), ("assignee", assignee), + ): + if not values: + continue + parts = [part.strip() for value in values for part in value.split(",")] + if any(not part for part in parts): + raise typer.BadParameter(f"--{key} needs a non-empty value") + if key == "source" and any(part not in _SOURCES for part in parts): + raise typer.BadParameter(f"--source must be one of: {', '.join(_SOURCES)}") + filters[key] = ",".join(dict.fromkeys(parts)) + # Reuse the CLI presets and RFC3339 validation. Compute each bound + # independently: an explicit --until must not cancel a relative --since. + if since in dates.SINCE_CHOICES or since is None: + opened_from, _ = resolve_dates(since, None, None) + else: + try: + opened_from = dates._validate_iso(since, "--since") + except ValueError as exc: + raise typer.BadParameter(str(exc)) from exc + try: + opened_to = dates._validate_iso(until, "--until") if until is not None else None + except ValueError as exc: + raise typer.BadParameter(str(exc)) from exc + if opened_from: + filters["opened_from"] = opened_from + if opened_to: + filters["opened_to"] = opened_to + if search is not None: + filters["q"] = search + return filters + + +def _filters_unavailable() -> None: + output.cli_warn( + "▲ Issue filters are not available on this FailproofAI Cloud yet; " + "Cloud may have returned unfiltered results. Upgrade your Cloud deployment." + ) + + def incidents_list( ctx: typer.Context, state_filter: Optional[str] = typer.Option(None, "--state", help="Filter by state(s): firing, acknowledged, resolved, closed (CSV)."), alert_id: Optional[str] = typer.Option(None, "--alert-id", help="Only incidents for this alert."), limit: int = typer.Option(50, "--limit", "-n", help="Max incidents to return."), show_id: bool = typer.Option(False, "--show-id", help="Show the full incident id instead of the short form (always full in --json)."), + agent: Optional[List[str]] = typer.Option(None, "--agent", help="Agent id (repeatable or CSV)."), + harness: Optional[List[str]] = typer.Option(None, "--harness", help="Harness (repeatable or CSV)."), + environment: Optional[List[str]] = typer.Option(None, "--env", help="Environment (repeatable or CSV)."), + machine: Optional[List[str]] = typer.Option(None, "--machine", help="Machine id (repeatable or CSV)."), + source: Optional[List[str]] = typer.Option(None, "--source", help="Source: alert, audit, manual (repeatable or CSV)."), + alert: Optional[List[str]] = typer.Option(None, "--alert", help="Alert id (repeatable or CSV)."), + audit: Optional[List[str]] = typer.Option(None, "--audit", help="Audit id (repeatable or CSV)."), + assignee: Optional[List[str]] = typer.Option(None, "--assignee", help="Email, me, or unassigned (repeatable or CSV)."), + since: Optional[str] = typer.Option(None, "--since", help="Opened since a preset (all, 15m, 1h, 6h, 24h, 7d) or RFC3339 timestamp."), + until: Optional[str] = typer.Option(None, "--until", help="Opened before an RFC3339 timestamp."), + search: Optional[str] = typer.Option(None, "--search", help="Text in the issue title or summary."), ) -> None: """List incidents in a boxed table (newest-opened first). @@ -93,8 +153,14 @@ def incidents_list( state: AppState = ctx.obj _validate_states(state_filter) validate_limit(limit) + filters = _issue_filters(agent, harness, environment, machine, source, alert, audit, assignee, since, until, search) + if alert_id and "alert_id" in filters: + raise typer.BadParameter("use --alert or --alert-id, not both") cctx = require_auth(state) - incidents = api.list_incidents(cctx, state=state_filter, alert_id=alert_id, limit=limit) + incidents = api.list_incidents( + cctx, state=state_filter, alert_id=alert_id, limit=limit, + filters=filters, on_filters_unavailable=_filters_unavailable, + ) if state.json: output.emit_json({"issues": incidents}) return @@ -105,6 +171,17 @@ def incidents_list( def incidents_count( ctx: typer.Context, state_filter: Optional[str] = typer.Option(None, "--state", help="Filter by state(s) (CSV)."), + agent: Optional[List[str]] = typer.Option(None, "--agent", help="Agent id (repeatable or CSV)."), + harness: Optional[List[str]] = typer.Option(None, "--harness", help="Harness (repeatable or CSV)."), + environment: Optional[List[str]] = typer.Option(None, "--env", help="Environment (repeatable or CSV)."), + machine: Optional[List[str]] = typer.Option(None, "--machine", help="Machine id (repeatable or CSV)."), + source: Optional[List[str]] = typer.Option(None, "--source", help="Source: alert, audit, manual (repeatable or CSV)."), + alert: Optional[List[str]] = typer.Option(None, "--alert", help="Alert id (repeatable or CSV)."), + audit: Optional[List[str]] = typer.Option(None, "--audit", help="Audit id (repeatable or CSV)."), + assignee: Optional[List[str]] = typer.Option(None, "--assignee", help="Email, me, or unassigned (repeatable or CSV)."), + since: Optional[str] = typer.Option(None, "--since", help="Opened since a preset (all, 15m, 1h, 6h, 24h, 7d) or RFC3339 timestamp."), + until: Optional[str] = typer.Option(None, "--until", help="Opened before an RFC3339 timestamp."), + search: Optional[str] = typer.Option(None, "--search", help="Text in the issue title or summary."), ) -> None: """Count incidents (optionally by state) as a compact stat card. @@ -117,14 +194,46 @@ def incidents_count( """ state: AppState = ctx.obj _validate_states(state_filter) + filters = _issue_filters(agent, harness, environment, machine, source, alert, audit, assignee, since, until, search) cctx = require_auth(state) - count = api.count_incidents(cctx, state=state_filter) + count = api.count_incidents(cctx, state=state_filter, filters=filters, on_filters_unavailable=_filters_unavailable) if state.json: output.emit_json({"count": count}) else: output.render_incident_count(count, state=state_filter) +def incidents_facets( + ctx: typer.Context, + state_filter: Optional[str] = typer.Option(None, "--state", help="Filter by state(s) (CSV)."), + agent: Optional[List[str]] = typer.Option(None, "--agent", help="Agent id (repeatable or CSV)."), + harness: Optional[List[str]] = typer.Option(None, "--harness", help="Harness (repeatable or CSV)."), + environment: Optional[List[str]] = typer.Option(None, "--env", help="Environment (repeatable or CSV)."), + machine: Optional[List[str]] = typer.Option(None, "--machine", help="Machine id (repeatable or CSV)."), + source: Optional[List[str]] = typer.Option(None, "--source", help="Source: alert, audit, manual (repeatable or CSV)."), + alert: Optional[List[str]] = typer.Option(None, "--alert", help="Alert id (repeatable or CSV)."), + audit: Optional[List[str]] = typer.Option(None, "--audit", help="Audit id (repeatable or CSV)."), + assignee: Optional[List[str]] = typer.Option(None, "--assignee", help="Email, me, or unassigned (repeatable or CSV)."), + since: Optional[str] = typer.Option(None, "--since", help="Opened since a preset (all, 15m, 1h, 6h, 24h, 7d) or RFC3339 timestamp."), + until: Optional[str] = typer.Option(None, "--until", help="Opened before an RFC3339 timestamp."), + search: Optional[str] = typer.Option(None, "--search", help="Text in the issue title or summary."), +) -> None: + """Show available issue filter values with counts. Needs issues:read. + + Example: + + * `fp --json issues facets --env prod` + """ + state: AppState = ctx.obj + _validate_states(state_filter) + filters = _issue_filters(agent, harness, environment, machine, source, alert, audit, assignee, since, until, search) + data = api.issue_facets(require_auth(state), state=state_filter, filters=filters) + if state.json: + output.emit_json(data) + else: + output.render_issue_facets(data) + + def incidents_show( ctx: typer.Context, incident_id: str = typer.Argument(..., help="Incident id."), @@ -671,10 +780,11 @@ def register(app: typer.Typer) -> None: no_args_is_help=True, rich_markup_mode="markdown", context_settings={"help_option_names": ["-h", "--help"]}, - help="Triage issues (list / count / show / ack / assign / resolve / close / archive / unarchive / clear / comment-* / subscribe* / open).", + help="Triage issues (list / count / facets / show / ack / assign / resolve / close / archive / unarchive / clear / comment-* / subscribe* / open).", ) inc.command("list", epilog=GLOBALS_EPILOG)(incidents_list) inc.command("count", epilog=GLOBALS_EPILOG)(incidents_count) + inc.command("facets", epilog=GLOBALS_EPILOG)(incidents_facets) inc.command("show", epilog=GLOBALS_EPILOG)(incidents_show) inc.command("ack", epilog=GLOBALS_EPILOG)(incidents_ack) inc.command("assign", epilog=GLOBALS_EPILOG)(incidents_assign) diff --git a/fp-cloud-cli/fp_cli/models.py b/fp-cloud-cli/fp_cli/models.py index c78ee8453..42bd3491d 100644 --- a/fp-cloud-cli/fp_cli/models.py +++ b/fp-cloud-cli/fp_cli/models.py @@ -430,6 +430,8 @@ class Incident: subscribers: Optional[List[Dict[str, Any]]] = None comments: Optional[List[Dict[str, Any]]] = None activity: Optional[List[Dict[str, Any]]] = None + #: Stored evidence scope. Older Clouds omit it; no event scan is needed. + scope: Dict[str, Any] = field(default_factory=dict) @classmethod def from_dict(cls, d: Dict[str, Any]) -> "Incident": @@ -461,6 +463,7 @@ def from_dict(cls, d: Dict[str, Any]) -> "Incident": subscribers=d.get("subscribers"), comments=d.get("comments"), activity=d.get("activity"), + scope=dict(d.get("scope") or {}), ) diff --git a/fp-cloud-cli/fp_cli/output.py b/fp-cloud-cli/fp_cli/output.py index 623ad12b2..a199b3965 100644 --- a/fp-cloud-cli/fp_cli/output.py +++ b/fp-cloud-cli/fp_cli/output.py @@ -603,7 +603,7 @@ def version_banner(version: str) -> None: # is one level down, in `fp audits --help`, which walks the real # Click tree. ("audits", "Schedule audits and triage their findings.", "list show create edit delete run runs findings finding ack mute dismiss resolve reopen assign context-* contracts"), - ("issues", "Triage and resolve issues.", "list count show ack assign resolve close archive unarchive clear comment-* subscribe subscribers unsubscribe open"), + ("issues", "Triage and resolve issues.", "list count facets show ack assign resolve close archive unarchive clear comment-* subscribe subscribers unsubscribe open"), ("settings", "View and change org settings.", "list schema set context-windows"), ("roles", "Manage permission sets.", "list show create update delete"), ("plan", "Inspect offers and team trial.", "claim"), @@ -4381,9 +4381,16 @@ def render_incidents(incidents: Sequence[Any], *, show_id: bool = False) -> None iid = (getattr(i, "id", "") or "-") if show_id else _short_id(getattr(i, "id", "") or "-") opened = _age_compact(getattr(i, "opened_at", "")) or "-" label = getattr(i, "title", None) or getattr(i, "alert_name", None) or "—" + identity = Text(label, style=theme.TEXT) + scope = getattr(i, "scope", None) or {} + # Keep tags with the title so narrow terminals never lose provenance + # by dropping a rightmost column. + for key, tag in (("harnesses", "harness"), ("agents", "agent")): + for value in scope.get(key, []): + identity.append(f" [{tag}:{value}]", style=theme.TEXT_DIM) rows.append([ Text(iid, style=theme.TEXT_DIM), - Text(label, style=theme.TEXT), + identity, _incident_source_cell(getattr(i, "source", None), getattr(i, "alert_name", None)), _severity_cell(getattr(i, "alert_severity", "") or "-"), _incident_status_cell(getattr(i, "state", "")), @@ -4397,6 +4404,26 @@ def render_incidents(incidents: Sequence[Any], *, show_id: bool = False) -> None empty_message="no issues", title=title) +def render_issue_facets(data: Dict[str, Any]) -> None: + """Available values with counts, without dumping a raw response in human mode.""" + facets = data.get("facets", data) + rows = [] + for facet, values in facets.items(): + if not isinstance(values, list): + continue + for item in values: + if isinstance(item, dict): + rows.append([ + Text(str(facet), style=theme.LABEL), + Text(str(item.get("label") or item.get("value", "—")), style=theme.TEXT), + Text(str(item.get("count", 0)), style=theme.TEXT_DIM), + ]) + render_list_panel( + "issue facets", header=["filter", "value", "count"], rows=rows, + days=set(), order=None, empty_message="no issue filter values", + ) + + def incidents_footer(incidents: Sequence[Any]) -> None: """Distribution summary under the incidents box (stderr): ``{total} incidents · firing {f} · acknowledged {a} · resolved {r}`` — each state count in its state colour, present only when diff --git a/fp-cloud-cli/tests/test_issue_filters.py b/fp-cloud-cli/tests/test_issue_filters.py new file mode 100644 index 000000000..f13441e46 --- /dev/null +++ b/fp-cloud-cli/tests/test_issue_filters.py @@ -0,0 +1,206 @@ +"""The issue-filter wire contract in both auth modes, including older Clouds.""" + +import json +from datetime import datetime, timezone + +import httpx +import pytest +import respx + +from fp_cli import client, dates, output +from fp_cli.app import app +from fp_cli.models import Incident + +BASE = "http://dash.test" +FILTER_FLAGS = [ + "--agent", "claude-repo", "--agent", "codex-repo", + "--harness", "claude", "--harness", "codex", + "--env", "prod", "--env", "staging", + "--machine", "m1", "--machine", "m2", + "--source", "alert", "--source", "audit", + "--alert", "a1", "--alert", "a2", + "--audit", "au1", "--audit", "au2", + "--assignee", "me", "--assignee", "unassigned", "--assignee", "a@example.test", + "--since", "2026-10-01T00:00:00Z", "--until", "2026-10-14T00:00:00Z", + "--search", "tool failures & latency", +] +FILTERS = { + "agent": "claude-repo,codex-repo", + "harness": "claude,codex", + "environment": "prod,staging", + "machine": "m1,m2", + "source": "alert,audit", + "alert_id": "a1,a2", + "audit_id": "au1,au2", + "assignee": "me,unassigned,a@example.test", + "opened_from": "2026-10-01T00:00:00Z", + "opened_to": "2026-10-14T00:00:00Z", + "q": "tool failures & latency", +} +FACETS = {"harnesses": [{"value": "claude", "count": 3}], + "agents": [{"value": "claude-repo", "count": 2}]} + + +@pytest.mark.parametrize("key_mode", [False, True]) +@pytest.mark.parametrize("command", ["list", "count", "facets"]) +@respx.mock +def test_all_filters_in_session_and_key_mode(logged_in, runner, key_mode, command): + prefix = "/v1" if key_mode else "/api" + path = f"{BASE}{prefix}/issues" + (f"/{command}" if command != "list" else "") + response = {"issues": [{"id": "i1", "scope": {"harnesses": ["claude"], "agents": ["claude-repo"]}}], + "filters": FILTERS} if command == "list" else ( + {"count": 3, "filters": FILTERS} if command == "count" else FACETS) + route = respx.get(path).respond(200, json=response) + auth = ["--api-key", "key"] if key_mode else [] + result = runner.invoke(app, ["--json", *auth, "issues", command, + "--state", "firing,acknowledged", *FILTER_FLAGS]) + assert result.exit_code == 0, result.output + expected = {**FILTERS, "state": "firing,acknowledged"} + if command == "list": + expected["limit"] = "50" + request = route.calls[0].request + assert dict(request.url.params) == expected + if key_mode: + assert request.headers["authorization"] == "Bearer key" + assert "cookie" not in request.headers + else: + assert "ae_session=tok" in request.headers["cookie"] + assert "authorization" not in request.headers + payload = json.loads(result.stdout) + if command == "list": + assert payload["issues"][0]["scope"]["harnesses"] == ["claude"] + elif command == "count": + assert payload == {"count": 3} + else: + assert payload == FACETS + assert len(respx.calls) == 1, "echoed filters need no capability probe" + + +@pytest.mark.parametrize("key_mode", [False, True]) +@pytest.mark.parametrize("command", ["list", "count"]) +@respx.mock +def test_old_cloud_warns_once_but_preserves_json(logged_in, runner, key_mode, command): + prefix = "/v1" if key_mode else "/api" + path = f"{BASE}{prefix}/issues" + ("/count" if command == "count" else "") + respx.get(path).respond(200, json=[{"id": "old"}] if command == "list" else {"count": 8}) + facets = respx.get(f"{BASE}{prefix}/issues/facets").respond(404, json={"error": "not found"}) + auth = ["--api-key", "key"] if key_mode else [] + result = runner.invoke(app, ["--json", "--quiet", *auth, "issues", command, + "--harness", "claude", "--env", "prod"]) + assert result.exit_code == 0, result.output + assert result.stderr.count("not available on this FailproofAI Cloud yet") == 1 + assert "unfiltered" in result.stderr + assert facets.call_count == 1 + payload = json.loads(result.stdout) + assert (payload["issues"][0]["id"] == "old") if command == "list" else payload == {"count": 8} + + +@pytest.mark.parametrize("key_mode", [False, True]) +@respx.mock +def test_missing_facets_has_existing_availability_error(logged_in, runner, key_mode): + prefix = "/v1" if key_mode else "/api" + respx.get(f"{BASE}{prefix}/issues/facets").respond(404, json={"error": "not found"}) + auth = ["--api-key", "key"] if key_mode else [] + result = runner.invoke(app, ["--json", *auth, "issues", "facets", "--env", "prod"]) + assert result.exit_code == 6, result.output + assert "not available on this FailproofAI Cloud yet" in result.stdout + json.loads(result.stdout) + + +@respx.mock +def test_bare_list_on_new_cloud_probes_facets_without_warning(logged_in, runner): + respx.get(f"{BASE}/api/issues").respond(200, json=[{"id": "i1"}]) + route = respx.get(f"{BASE}/api/issues/facets").respond(200, json=FACETS) + result = runner.invoke(app, ["--json", "issues", "list", "--source", "manual"]) + assert result.exit_code == 0, result.output + assert dict(route.calls[0].request.url.params) == {"source": "manual"} + assert "unfiltered" not in result.stderr + + +@respx.mock +def test_partial_echo_still_checks_capability(logged_in, runner): + respx.get(f"{BASE}/api/issues").respond(200, json={"issues": [], "filters": {"harness": "claude"}}) + route = respx.get(f"{BASE}/api/issues/facets").respond(404, json={}) + result = runner.invoke(app, ["--json", "issues", "list", "--harness", "claude", "--env", "prod"]) + assert result.exit_code == 0 + assert route.call_count == 1 + assert "unfiltered" in result.stderr + + +@pytest.mark.parametrize("status", [401, 403, 500]) +@respx.mock +def test_probe_errors_are_not_disguised_as_older_cloud(logged_in, runner, status): + respx.get(f"{BASE}/api/issues/count").respond(200, json={"count": 8}) + respx.get(f"{BASE}/api/issues/facets").respond(status, json={"error": "probe failed"}) + result = runner.invoke(app, ["--json", "issues", "count", "--harness", "claude"]) + assert result.exit_code != 0 + assert "unfiltered" not in result.stderr + + +@pytest.mark.parametrize("command", ["list", "count", "facets"]) +@pytest.mark.parametrize("flags", [ + ["--source", "invalid"], ["--harness", ""], ["--agent", "a,,b"], + ["--since", "yesterday"], ["--until", "2026-10-14"], +]) +@respx.mock +def test_invalid_filters_fail_before_http(logged_in, runner, command, flags): + result = runner.invoke(app, ["issues", command, *flags]) + assert result.exit_code == 2, result.output + assert not respx.calls + + +@respx.mock +def test_relative_since_keeps_until_and_csv_is_normalized(logged_in, runner, monkeypatch): + real = dates.resolve_range + now = datetime(2026, 10, 14, tzinfo=timezone.utc) + monkeypatch.setattr(dates, "resolve_range", lambda *args: real(*args, now=now)) + expected = {"harness": "claude,codex", "opened_from": "2026-10-07T00:00:00Z", + "opened_to": "2026-10-14T00:00:00Z"} + route = respx.get(f"{BASE}/api/issues/count").respond(200, json={"count": 0, "applied_filters": expected}) + result = runner.invoke(app, ["--json", "issues", "count", + "--harness", "claude, codex", "--harness", "claude", + "--since", "7d", "--until", "2026-10-14T00:00:00Z"]) + assert result.exit_code == 0, result.output + assert dict(route.calls[0].request.url.params) == expected + assert len(respx.calls) == 1 + + +@respx.mock +def test_existing_alert_id_path_is_kept_and_combined_filters_use_issues(logged_in, runner): + legacy = respx.get(f"{BASE}/api/alerts/a1/issues").respond(200, json=[]) + result = runner.invoke(app, ["--json", "issues", "list", "--alert-id", "a1"]) + assert result.exit_code == 0, result.output + assert legacy.called + route = respx.get(f"{BASE}/api/issues").respond(200, json={"issues": [], "filters": {"harness": "codex"}}) + result = runner.invoke(app, ["--json", "issues", "list", "--alert-id", "a1", "--harness", "codex"]) + assert result.exit_code == 0, result.output + assert dict(route.calls[0].request.url.params) == {"limit": "50", "harness": "codex", "alert_id": "a1"} + + +def test_human_rows_include_harness_and_agent_tags(capsys): + output.render_incidents([Incident.from_dict({ + "id": "i1", "title": "Timeouts", + "scope": {"harnesses": ["claude"], "agents": ["claude-repo"]}, + })]) + out = capsys.readouterr().out + assert "[harness:claude]" in out + assert "[agent:claude-repo]" in out + + +def test_human_facets_show_values_counts_and_empty_state(capsys): + output.render_issue_facets(FACETS) + out = capsys.readouterr().out + assert "claude" in out and "3" in out and "claude-repo" in out + assert "{'value'" not in out + output.render_issue_facets({}) + assert "no issue filter values" in capsys.readouterr().out + + +def test_programmatic_client_reports_missing_filters_without_printing(): + def handle(request): + return httpx.Response(404, json={}) if request.url.path.endswith("/facets") else httpx.Response(200, json=[]) + ctx = client.ClientContext(BASE, token="tok", transport=httpx.MockTransport(handle)) + warnings = [] + assert client.list_incidents(ctx, filters={"harness": "codex"}, + on_filters_unavailable=lambda: warnings.append(True)) == [] + assert warnings == [True] From e22360b35dac2b8e4ee76a409e2be8a614c75344 Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:05:14 +0000 Subject: [PATCH 3/6] docs(issues): document filters and verify the stacked feature in CI --- .github/workflows/issue-filters.yml | 69 +++++++++++ CHANGELOG.md | 6 + __tests__/ci/issue-filters-workflow.test.ts | 39 ++++++ fp-cloud-cli/CHANGELOG.md | 6 + fp-cloud-cli/README.md | 27 +++++ fp-cloud-cli/skill/SKILL.md | 6 + fp-cloud-cli/skill/references/commands.md | 16 +++ fp-cloud-cli/tests/issue_filters_smoke.py | 124 ++++++++++++++++++++ 8 files changed, 293 insertions(+) create mode 100644 .github/workflows/issue-filters.yml create mode 100644 __tests__/ci/issue-filters-workflow.test.ts create mode 100644 fp-cloud-cli/tests/issue_filters_smoke.py diff --git a/.github/workflows/issue-filters.yml b/.github/workflows/issue-filters.yml new file mode 100644 index 000000000..d71af3394 --- /dev/null +++ b/.github/workflows/issue-filters.yml @@ -0,0 +1,69 @@ +name: Issue filters + +# The existing CI and parity workflows do not target this feature's stacked +# base. Keep verification separate from the workflow edit owned by feat/otel. +on: + pull_request: + branches: [feat/fp-cli-parity] + paths: + - "crates/fpai-collect/**" + - "fp-cloud-cli/**" + - ".github/workflows/issue-filters.yml" + - "CHANGELOG.md" + +permissions: + contents: read + +concurrency: + group: issue-filters-${{ github.ref }} + cancel-in-progress: true + +jobs: + collector: + name: Collector harness + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.1" + components: rustfmt, clippy + - run: cargo fmt --all -- --check + - run: cargo clippy -p fpai-collect --all-targets -- -D warnings + - run: cargo test -p fpai-collect + + cloud-cli: + name: Issue filters (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + defaults: + run: + working-directory: fp-cloud-cli + steps: + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + cache-dependency-glob: fp-cloud-cli/uv.lock + - run: uv sync --locked --extra dev --python ${{ matrix.python-version }} + - name: Test all CLI behaviour + env: + FP_CLI_REQUIRE_CONTRACT: "1" + run: uv run --extra dev pytest -q + - run: uv build + - name: Clean wheel install and user-level issue-filter smoke + run: | + uv venv /tmp/fp-issue-smoke --python ${{ matrix.python-version }} + uv pip install --python /tmp/fp-issue-smoke/bin/python dist/*.whl + /tmp/fp-issue-smoke/bin/fp --version + /tmp/fp-issue-smoke/bin/python tests/issue_filters_smoke.py \ + --fp /tmp/fp-issue-smoke/bin/fp --artifacts /tmp/fp-issue-artifacts diff --git a/CHANGELOG.md b/CHANGELOG.md index 8284cf57c..7a18db226 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 1.0.11-beta.1 — 2026-10-14 + +### Features + +- Stamp harness provenance on every collected event, using the source kind, the hook row's CLI, or sdk for SDK batches, without replacing existing values; add Cloud CLI issue filters and facets (PR pending). + ## 1.0.11-beta.1 — 2026-10-13 ### Features diff --git a/__tests__/ci/issue-filters-workflow.test.ts b/__tests__/ci/issue-filters-workflow.test.ts new file mode 100644 index 000000000..69c7bcb22 --- /dev/null +++ b/__tests__/ci/issue-filters-workflow.test.ts @@ -0,0 +1,39 @@ +// @vitest-environment node +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; + +const workflow = parse(readFileSync( + resolve(process.cwd(), ".github/workflows/issue-filters.yml"), "utf8", +)); +const scripts = (job: { steps: { run?: string }[] }) => + job.steps.map((step) => step.run ?? "").join("\n"); + +describe("issue-filter stacked verification", () => { + it("tests the actual PR base without publishing or release credentials", () => { + expect(workflow.on.pull_request.branches).toContain("feat/fp-cli-parity"); + expect(workflow.on.workflow_dispatch).toBeUndefined(); + expect(workflow.permissions).toEqual({ contents: "read" }); + expect(workflow.on.pull_request.paths).toContain("crates/fpai-collect/**"); + expect(workflow.on.pull_request.paths).toContain("fp-cloud-cli/**"); + }); + + it("runs collector formatting, clippy and all cargo tests", () => { + const run = scripts(workflow.jobs.collector); + expect(run).toContain("cargo fmt --all -- --check"); + expect(run).toContain("cargo clippy -p fpai-collect --all-targets -- -D warnings"); + expect(run).toContain("cargo test -p fpai-collect"); + }); + + it("tests both auth modes across supported Python versions and a clean wheel", () => { + const job = workflow.jobs["cloud-cli"]; + expect(job.strategy.matrix["python-version"]).toEqual(["3.10", "3.11", "3.12", "3.13"]); + const run = scripts(job); + expect(run).toContain("uv sync --locked --extra dev"); + expect(run).toContain("pytest -q"); + expect(run).toContain("uv build"); + expect(run).toContain("uv pip install --python /tmp/fp-issue-smoke/bin/python dist/*.whl"); + expect(run).toContain("tests/issue_filters_smoke.py"); + }); +}); diff --git a/fp-cloud-cli/CHANGELOG.md b/fp-cloud-cli/CHANGELOG.md index d028972fc..5b6d96345 100644 --- a/fp-cloud-cli/CHANGELOG.md +++ b/fp-cloud-cli/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog — `fp` CLI +## 0.0.2b1 — 2026-10-14 + +### Added + +- Filter issues list/count and discover facets by agent, harness, environment, machine, source/alert/audit, assignee, opened dates and title/summary text in session or key mode; show scope tags and warn when an older Cloud may ignore filters (PR pending). + ## 0.0.2b1 — 2026-10-13 ### Added diff --git a/fp-cloud-cli/README.md b/fp-cloud-cli/README.md index 227b29a1f..24e30b4c5 100644 --- a/fp-cloud-cli/README.md +++ b/fp-cloud-cli/README.md @@ -127,6 +127,33 @@ fp --json events --session-id run-001 --all | jq '.events[].payload' fp --json sessions --since 7d --fields session_id,status,scores ``` +### Filter the failures inbox + +Issue filters work with either a signed-in session or an API key carrying `issues:read`: + +```bash +fp --json issues facets --env prod +fp issues list --harness claude --harness codex --env prod --assignee me --since 7d +fp --json issues count --source audit --audit weekly-audit --machine ci-runner-01 +fp issues list --agent claude-checkout --assignee unassigned --search "tool timeout" \ + --since 2026-10-01T00:00:00Z --until 2026-10-14T00:00:00Z +``` + +`issues list`, `count` and `facets` share `--agent`, `--harness`, `--env`, +`--machine`, `--source alert|audit|manual`, `--alert`, `--audit` and +`--assignee EMAIL|me|unassigned`. These accept repeated flags or comma-separated +values (OR within a filter, AND between filters). `--since` accepts the existing +`all|15m|1h|6h|24h|7d` presets or RFC3339; `--until` accepts RFC3339. Dates apply +to issue opening, and `--search` searches title + summary. Human rows show the +stored harness/agent scope; facets show available values and counts. + +Older Clouds may ignore the new query parameters. If no filters are echoed, +the CLI probes facets; a missing route produces one warning on stderr that +list/count results may be unfiltered, leaving stdout JSON usable. Calling +missing `facets` directly exits 6 with “not available on this FailproofAI Cloud yet”. +Upgrade Cloud for filtering; the collector's new `harness` event field is +additive and ignored by old servers. Existing `list --alert-id` remains supported. + ### Cloud-managed policies Three commands for the three jobs the dashboard splits across three pages — diff --git a/fp-cloud-cli/skill/SKILL.md b/fp-cloud-cli/skill/SKILL.md index 146f14964..882c738e5 100644 --- a/fp-cloud-cli/skill/SKILL.md +++ b/fp-cloud-cli/skill/SKILL.md @@ -187,6 +187,12 @@ you need a flag you don't already know. - `errors` — errored events; `--aggregate` for count / sessions / agents / last-seen. - `usage` — current org usage for its fixed 30-day metering window; needs `usage:read`. - `list ` — **discover valid filter values first**: `envs agents event_types score_filters models hooks tools error_types`. +- `issues list|count|facets` — scope the failures inbox by agent/harness/environment/machine, + source or a specific alert/audit, assignee, opened dates and title/summary text. Use + `fp --json issues facets --env prod` to discover values/counts, then + `fp --json issues list --env prod --harness claude --assignee me --since 7d`. + Full flags are in `references/commands.md`. If stderr warns that Cloud may have + returned unfiltered results, do not treat the response as proof of the requested scope. **Manage (permission-gated, mutations):** - `keys list|show|create|update|disable|regenerate` — API keys; secret shown once. diff --git a/fp-cloud-cli/skill/references/commands.md b/fp-cloud-cli/skill/references/commands.md index 6d4ff47cb..7e5b2c629 100644 --- a/fp-cloud-cli/skill/references/commands.md +++ b/fp-cloud-cli/skill/references/commands.md @@ -185,6 +185,22 @@ The title column truncates to whatever width is left so the fixed columns always The single board for everything needing human attention — alert breaches (`source: alert`), hand-raised issues (`manual`), and audit findings (`audit`). Referenced by id (short ids accepted, `--show-id` shows them). This group was **renamed from `incidents`**; the old name no longer exists. Reads and ack/comment need `issues:read`; opening, assigning, and subscribing others need `issues:create`; resolving, closing, archiving and clearing need `issues:close` (and `clear` also needs `audits:write`, because it resolves the findings behind the issues). - `issues list [--state firing|acknowledged|resolved|closed] [--alert-id ] [--limit N] [--show-id]` — there is **no** `--severity` filter on this command. Archived issues are hidden. - `issues count` +- `issues facets` — available filter values with counts; read-only, needs `issues:read`. + +`list`, `count` and `facets` share filters (session and API-key mode): +`--agent`, `--harness`, `--env`, `--machine`, `--source alert|audit|manual`, +`--alert `, `--audit `, `--assignee EMAIL|me|unassigned`. +Each accepts repeated flags or comma-separated values. Different dimensions combine +with AND; values within one dimension combine with OR. `--since` accepts the existing +`all|15m|1h|6h|24h|7d` presets or an RFC3339 timestamp; `--until` accepts RFC3339. +Dates filter when the issue **opened**, not when it last fired. `--search TEXT` +searches title + summary. Human issue rows include harness/agent tags from stored scope. +Global `--json` goes before `issues`, and facets JSON preserves the server response. +On an older Cloud, missing facets exits 6 with “not available on this FailproofAI Cloud +yet”. Filtered list/count without a filter echo probe facets; a missing route prints +one stderr warning that results may be unfiltered (also under `--json`/`--quiet`). +The legacy `list --alert-id` stays supported; use `--alert` for multi-alert filtering. + - `issues show ` — identity + comments + subscribers + **activity log** (read this before acting). An audit-born issue (`source: audit`) carries the full finding it graduated from and back-links to the audit/run. - `issues ack ` · `issues assign --assignee ` (repeatable; omit to clear all assignees; each must be an operator) · `issues resolve ` (calm confirm). **On an audit issue these stay in sync with the finding** — resolving the issue resolves the underlying audit finding, so it can't reappear on the next run (equivalently, triage it with `audits resolve `; both surfaces agree). - `issues close ` (calm confirm) — the second terminal state: **"we're done with it", not "we fixed it"** (won't fix / not a problem / stale). The difference from `resolve` is what happens next: a **resolved** issue REOPENS when its audit finding recurs, a **closed** one does not. On an audit issue this marks the finding `dismissed`; it does **not** write the org-wide fingerprint suppression that `audits dismiss` writes, so closing one issue never hides that pattern in your other audits. Exit 9 if the issue already ended — an issue ends once, and neither terminal state may overwrite the other. diff --git a/fp-cloud-cli/tests/issue_filters_smoke.py b/fp-cloud-cli/tests/issue_filters_smoke.py new file mode 100644 index 000000000..108eaa82b --- /dev/null +++ b/fp-cloud-cli/tests/issue_filters_smoke.py @@ -0,0 +1,124 @@ +"""Exercise an installed fp against a local Cloud, never a production deployment.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import tempfile +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, urlsplit + + +class Handler(BaseHTTPRequestHandler): + legacy = False + calls = [] + + def log_message(self, *args): + pass + + def do_GET(self): + parsed = urlsplit(self.path) + key_mode = parsed.path.startswith("/v1/") + if key_mode: + assert self.headers.get("Authorization") == "Bearer fake-key" + assert not self.headers.get("Cookie") + else: + assert "ae_session=fake-session" in self.headers.get("Cookie", "") + assert not self.headers.get("Authorization") + filters = {key: values[0] for key, values in parse_qs(parsed.query).items() + if key not in ("limit", "state")} + self.calls.append({"path": parsed.path, "filters": filters}) + route = parsed.path.removeprefix("/v1").removeprefix("/api") + issue = {"id": "i1", "title": "Tool timeouts", "state": "firing", + "scope": {"harnesses": ["claude"], "agents": ["claude-repo"]}} + status = 200 + if route == "/issues/facets": + status = 404 if self.legacy else 200 + data = {"error": "not found"} if self.legacy else { + "harnesses": [{"value": "claude", "count": 2}], + "agents": [{"value": "claude-repo", "count": 2}], + } + elif route == "/issues/count": + data = {"count": 2} if self.legacy else {"count": 2, "filters": filters} + elif route == "/issues": + data = [issue] if self.legacy else {"issues": [issue], "filters": filters} + else: + status, data = 404, {"error": "unknown route"} + raw = json.dumps(data).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--fp", required=True) + parser.add_argument("--artifacts", required=True) + args = parser.parse_args() + artifacts = Path(args.artifacts) + artifacts.mkdir(parents=True, exist_ok=True) + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + with tempfile.TemporaryDirectory(prefix="fp-issue-smoke-") as cli_home: + env = {key: value for key, value in os.environ.items() + if not key.startswith("FP_") and key != "FAILPROOFAI_HOME"} + env.update(FP_HOME=cli_home, FP_NO_ANALYTICS="1", TERM="dumb", COLUMNS="140") + for key_mode in (False, True): + auth = ["--api-key", "fake-key"] if key_mode else ["--token", "fake-session"] + mode = "key" if key_mode else "session" + for command in ("list", "count", "facets"): + for as_json in (False, True): + result = subprocess.run([ + args.fp, "--base-url", f"http://127.0.0.1:{server.server_port}", + *auth, *(["--json"] if as_json else []), "issues", command, + "--harness", "claude", "--harness", "codex", "--env", "prod", + "--assignee", "unassigned", "--since", "2026-10-01T00:00:00Z", + "--until", "2026-10-14T00:00:00Z", "--search", "tool timeout", + ], env=env, text=True, capture_output=True, timeout=20) + assert result.returncode == 0, result.stderr + if as_json: + data = json.loads(result.stdout) + assert data["count"] == 2 if command == "count" else bool(data) + elif command == "list": + assert "[harness:claude]" in result.stdout + assert "[agent:claude-repo]" in result.stdout + elif command == "facets": + assert "claude" in result.stdout and "2" in result.stdout + assert "unfiltered" not in result.stderr + stem = f"issues-{mode}-{command}-{'json' if as_json else 'human'}" + (artifacts / f"{stem}.txt").write_text(result.stdout + result.stderr) + assert Handler.calls[-1]["filters"]["harness"] == "claude,codex" + Handler.legacy = True + for command in ("list", "count", "facets"): + result = subprocess.run([ + args.fp, "--base-url", f"http://127.0.0.1:{server.server_port}", + *auth, "--json", "issues", command, "--harness", "claude", + ], env=env, text=True, capture_output=True, timeout=20) + json.loads(result.stdout) + if command == "facets": + assert result.returncode == 6 + assert "not available on this FailproofAI Cloud yet" in result.stdout + else: + assert result.returncode == 0 + assert result.stderr.count("unfiltered") == 1 + (artifacts / f"issues-{mode}-{command}-legacy.txt").write_text( + result.stdout + result.stderr) + Handler.legacy = False + (artifacts / "issue-filter-requests.json").write_text(json.dumps(Handler.calls, indent=2)) + finally: + server.shutdown() + server.server_close() + thread.join() + print("Installed fp: session/key filters, facets, human tags, JSON and older-Cloud checks passed.") + + +if __name__ == "__main__": + main() From dc3b64b1b23d738fb5ee7558020c591cf16aca8f Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:12:54 +0000 Subject: [PATCH 4/6] docs(changelog): link issue-filter PR and consolidate current release notes --- CHANGELOG.md | 6 +----- fp-cloud-cli/CHANGELOG.md | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a18db226..654aae139 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,7 @@ ### Features -- Stamp harness provenance on every collected event, using the source kind, the hook row's CLI, or sdk for SDK batches, without replacing existing values; add Cloud CLI issue filters and facets (PR pending). - -## 1.0.11-beta.1 — 2026-10-13 - -### Features +- Stamp harness provenance on every collected event, using the source kind, the hook row's CLI, or sdk for SDK batches, without replacing existing values; add Cloud CLI issue filters and facets (#900). - Cloud CLI parity: hosted code, LLM-judge and Jev evaluator authoring/deployment, dashboards, sentiment, assistant artifacts/approvals, org controls and the shared mint/pink screen language (#885) diff --git a/fp-cloud-cli/CHANGELOG.md b/fp-cloud-cli/CHANGELOG.md index 5b6d96345..693ce02fb 100644 --- a/fp-cloud-cli/CHANGELOG.md +++ b/fp-cloud-cli/CHANGELOG.md @@ -4,11 +4,7 @@ ### Added -- Filter issues list/count and discover facets by agent, harness, environment, machine, source/alert/audit, assignee, opened dates and title/summary text in session or key mode; show scope tags and warn when an older Cloud may ignore filters (PR pending). - -## 0.0.2b1 — 2026-10-13 - -### Added +- Filter issues list/count and discover facets by agent, harness, environment, machine, source/alert/audit, assignee, opened dates and title/summary text in session or key mode; show scope tags and warn when an older Cloud may ignore filters (#900). - Hosted evaluator authoring for code, LLM judge and Jev: create/deploy, AI draft specs, dry-run, labels, toggles, instances, schema, jobs and code-only backfill. - Dashboards and tiles; session overview/export/history/re-evaluate; latency and sentiment; query composition/export; alert recipients/issues and agent contracts. From c1875adfdfb924fd7678454dd7be785e379d6cc2 Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:39:40 +0000 Subject: [PATCH 5/6] fix(issues): align scope, facets and detection with Cloud server contract --- CHANGELOG.md | 2 + crates/fpai-collect/src/spool.rs | 1 + fp-cloud-cli/CHANGELOG.md | 2 + fp-cloud-cli/README.md | 8 +- fp-cloud-cli/fp_cli/client.py | 48 +++++--- fp-cloud-cli/fp_cli/models.py | 14 ++- fp-cloud-cli/fp_cli/output.py | 12 +- fp-cloud-cli/skill/references/commands.md | 10 +- fp-cloud-cli/tests/issue_filters_smoke.py | 28 ++++- fp-cloud-cli/tests/test_issue_filters.py | 136 +++++++++++++++++----- 10 files changed, 200 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 654aae139..1d514ed74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,8 @@ ### Fixes +- Align Cloud CLI issue tags, JSON and alert/audit facets with the server's top-level scope contract, without extra requests on normal filtered calls (#900). + - `failproofai policies` showed nothing for a machine still on the pre-pack builtins, while those builtins were enforcing. It now says how many under BUILT IN, with the command that moves them into a pack (#881) - `failproofai config --disconnect` told you to restart the daemon to stop it sending. It stops on its own within a few seconds, because it polls the credential, and now says so (#881) - A config change that landed while the daemon was restarting its collector after a backfill was recorded as already running and never applied. `failproofai config` re-adding an agent hits exactly that, writing its backfill request and then the selection, so the agent stayed off until some unrelated edit (#881) diff --git a/crates/fpai-collect/src/spool.rs b/crates/fpai-collect/src/spool.rs index c1173ce17..2d195a250 100644 --- a/crates/fpai-collect/src/spool.rs +++ b/crates/fpai-collect/src/spool.rs @@ -266,6 +266,7 @@ impl SpoolWriter { harness: &str, ) -> std::io::Result<()> { stamp_harness(&mut event, harness); + // The later stamp_harness(prefix) in push is a no-op: the key is already set. self.push(event).await } diff --git a/fp-cloud-cli/CHANGELOG.md b/fp-cloud-cli/CHANGELOG.md index 693ce02fb..bb92d9870 100644 --- a/fp-cloud-cli/CHANGELOG.md +++ b/fp-cloud-cli/CHANGELOG.md @@ -19,6 +19,8 @@ ### Fixes +- Align issue scope JSON and named alert/audit facets with agenteye #1074; avoid capability requests on normal filtered calls and limit empty-result probes and older-Cloud warnings to once per process (#900). + - Remove the unused colour-keyed permission rank, document full-id rendering, emit UTC `Z` timestamps for deploy backfills, and report approval-answer failures without a traceback while restoring the terminal (#885). - Reject natural-language binary judge steps, distinguish unknown routes from missing records on 404, normalize standalone backfill ranges to UTC `Z`, expose all subcommands in help, and render artifact headers before titles with consistent session-only wording (#885). - New Cloud commands use deliberate human summaries/tables and actionable empty states at 80/140 columns without raw response blobs. Evaluator previews display before writes and require explicit `--force` after failures; plan/claim deny key mode, deploy specs can omit the positional key, and older settings-route 404s degrade clearly. Existing JSON payloads stay unchanged (#885). diff --git a/fp-cloud-cli/README.md b/fp-cloud-cli/README.md index 24e30b4c5..70483248e 100644 --- a/fp-cloud-cli/README.md +++ b/fp-cloud-cli/README.md @@ -147,9 +147,11 @@ values (OR within a filter, AND between filters). `--since` accepts the existing to issue opening, and `--search` searches title + summary. Human rows show the stored harness/agent scope; facets show available values and counts. -Older Clouds may ignore the new query parameters. If no filters are echoed, -the CLI probes facets; a missing route produces one warning on stderr that -list/count results may be unfiltered, leaving stdout JSON usable. Calling +Older Clouds may ignore the new query parameters. A returned issue's top-level +`harnesses` key confirms filter support without another request. Non-empty +legacy rows warn once; empty lists or zero counts probe facets at most once +per process, and a missing route warns once that results may be unfiltered. +Positive counts make no extra request. Warnings leave stdout JSON usable. Calling missing `facets` directly exits 6 with “not available on this FailproofAI Cloud yet”. Upgrade Cloud for filtering; the collector's new `harness` event field is additive and ignored by old servers. Existing `list --alert-id` remains supported. diff --git a/fp-cloud-cli/fp_cli/client.py b/fp-cloud-cli/fp_cli/client.py index 4716a46de..d2ff81a5a 100644 --- a/fp-cloud-cli/fp_cli/client.py +++ b/fp-cloud-cli/fp_cli/client.py @@ -1294,6 +1294,12 @@ def test_alert(ctx: ClientContext, alert_id: str, channels: Optional[List[Dict[s # --- Incidents -------------------------------------------------------------- +# Capability detection is only a fallback for empty results. Keep both the +# probe and the compatibility warning to at most once per CLI process. +_issue_facets_probed = False +_issue_facets_available: Optional[bool] = None +_issue_filters_warned = False + def list_incidents( ctx: ClientContext, @@ -1314,8 +1320,8 @@ def list_incidents( if alert_id: params["alert_id"] = alert_id data = cloud_request(ctx, "GET", "/api/issues", params=params) - _check_issue_filters(ctx, data, filters, on_filters_unavailable) items = data.get("issues", []) if isinstance(data, dict) else data + _check_issue_filters(ctx, items, filters, on_filters_unavailable) return [Incident.from_dict(i) for i in (items if isinstance(items, list) else [])] @@ -1350,25 +1356,35 @@ def _check_issue_filters( ctx: ClientContext, data: Any, filters: Optional[Dict[str, str]], on_unavailable: Optional[Callable[[], None]], ) -> None: - """An echo proves support; otherwise probe facets, never infer it from rows. + """New Cloud always sends `harnesses`, even when its array is empty. - Older deployments return a bare, unfiltered array/count. Only a missing - capability route warrants the compatibility warning; auth/network/server - failures must remain visible, not be misreported as an older Cloud. + Non-empty lists confirm support from the raw rows (before model defaults + fill missing keys), or warn for legacy rows without making another request. + Empty lists/zero counts probe facets once per process. Positive counts have + no scope marker and make no extra request; a cached missing route still + warrants a warning. Probe auth/network/server errors remain visible. """ + global _issue_facets_probed, _issue_facets_available, _issue_filters_warned if not filters: return - echoed = data.get("filters", data.get("applied_filters")) if isinstance(data, dict) else None - if isinstance(echoed, dict) and all( - echoed.get(key) == value or echoed.get(key) == value.split(",") - for key, value in filters.items() - ): - return - try: - issue_facets(ctx, filters=filters) - except NotFoundError: - if on_unavailable: - on_unavailable() + if isinstance(data, list) and data: + if any(isinstance(row, dict) and "harnesses" in row for row in data): + return + unavailable = True + else: + empty = data == [] or isinstance(data, dict) and data.get("count") == 0 + if empty and not _issue_facets_probed: + _issue_facets_probed = True + try: + issue_facets(ctx, filters=filters) + except NotFoundError: + _issue_facets_available = False + else: + _issue_facets_available = True + unavailable = _issue_facets_available is False + if unavailable and on_unavailable and not _issue_filters_warned: + _issue_filters_warned = True + on_unavailable() def get_incident(ctx: ClientContext, incident_id: str) -> Incident: diff --git a/fp-cloud-cli/fp_cli/models.py b/fp-cloud-cli/fp_cli/models.py index 42bd3491d..0e0e75dcd 100644 --- a/fp-cloud-cli/fp_cli/models.py +++ b/fp-cloud-cli/fp_cli/models.py @@ -430,8 +430,12 @@ class Incident: subscribers: Optional[List[Dict[str, Any]]] = None comments: Optional[List[Dict[str, Any]]] = None activity: Optional[List[Dict[str, Any]]] = None - #: Stored evidence scope. Older Clouds omit it; no event scan is needed. - scope: Dict[str, Any] = field(default_factory=dict) + #: Stored evidence scope, returned as top-level fields by Cloud. + agent_ids: List[str] = field(default_factory=list) + harnesses: List[str] = field(default_factory=list) + environments: List[str] = field(default_factory=list) + machine_ids: List[str] = field(default_factory=list) + scope_computed_at: Optional[str] = None @classmethod def from_dict(cls, d: Dict[str, Any]) -> "Incident": @@ -463,7 +467,11 @@ def from_dict(cls, d: Dict[str, Any]) -> "Incident": subscribers=d.get("subscribers"), comments=d.get("comments"), activity=d.get("activity"), - scope=dict(d.get("scope") or {}), + agent_ids=list(d.get("agent_ids") or []), + harnesses=list(d.get("harnesses") or []), + environments=list(d.get("environments") or []), + machine_ids=list(d.get("machine_ids") or []), + scope_computed_at=d.get("scope_computed_at"), ) diff --git a/fp-cloud-cli/fp_cli/output.py b/fp-cloud-cli/fp_cli/output.py index a199b3965..da47db2b9 100644 --- a/fp-cloud-cli/fp_cli/output.py +++ b/fp-cloud-cli/fp_cli/output.py @@ -4382,11 +4382,10 @@ def render_incidents(incidents: Sequence[Any], *, show_id: bool = False) -> None opened = _age_compact(getattr(i, "opened_at", "")) or "-" label = getattr(i, "title", None) or getattr(i, "alert_name", None) or "—" identity = Text(label, style=theme.TEXT) - scope = getattr(i, "scope", None) or {} # Keep tags with the title so narrow terminals never lose provenance # by dropping a rightmost column. - for key, tag in (("harnesses", "harness"), ("agents", "agent")): - for value in scope.get(key, []): + for key, tag in (("harnesses", "harness"), ("agent_ids", "agent")): + for value in getattr(i, key, None) or []: identity.append(f" [{tag}:{value}]", style=theme.TEXT_DIM) rows.append([ Text(iid, style=theme.TEXT_DIM), @@ -4413,9 +4412,14 @@ def render_issue_facets(data: Dict[str, Any]) -> None: continue for item in values: if isinstance(item, dict): + label = ( + item.get("name") or item.get("id", "—") + if facet in ("alerts", "audits") + else item.get("value", "—") + ) rows.append([ Text(str(facet), style=theme.LABEL), - Text(str(item.get("label") or item.get("value", "—")), style=theme.TEXT), + Text(str(label), style=theme.TEXT), Text(str(item.get("count", 0)), style=theme.TEXT_DIM), ]) render_list_panel( diff --git a/fp-cloud-cli/skill/references/commands.md b/fp-cloud-cli/skill/references/commands.md index 7e5b2c629..37989bc1b 100644 --- a/fp-cloud-cli/skill/references/commands.md +++ b/fp-cloud-cli/skill/references/commands.md @@ -194,11 +194,15 @@ Each accepts repeated flags or comma-separated values. Different dimensions comb with AND; values within one dimension combine with OR. `--since` accepts the existing `all|15m|1h|6h|24h|7d` presets or an RFC3339 timestamp; `--until` accepts RFC3339. Dates filter when the issue **opened**, not when it last fired. `--search TEXT` -searches title + summary. Human issue rows include harness/agent tags from stored scope. +searches title + summary. Human issue rows read harness/agent tags from top-level +`harnesses`/`agent_ids`; `--json` also keeps `environments`, `machine_ids` and +`scope_computed_at`. Facets render alert/audit names; JSON retains their ids. Global `--json` goes before `issues`, and facets JSON preserves the server response. On an older Cloud, missing facets exits 6 with “not available on this FailproofAI Cloud -yet”. Filtered list/count without a filter echo probe facets; a missing route prints -one stderr warning that results may be unfiltered (also under `--json`/`--quiet`). +yet”. A returned row's `harnesses` key confirms list support without another request. +Non-empty legacy rows warn once. Empty lists/zero counts probe facets at most once +per process and warn once if missing (also under `--json`/`--quiet`); positive +counts make no extra request. The legacy `list --alert-id` stays supported; use `--alert` for multi-alert filtering. - `issues show ` — identity + comments + subscribers + **activity log** (read this before acting). An audit-born issue (`source: audit`) carries the full finding it graduated from and back-links to the audit/run. diff --git a/fp-cloud-cli/tests/issue_filters_smoke.py b/fp-cloud-cli/tests/issue_filters_smoke.py index 108eaa82b..1cac68d80 100644 --- a/fp-cloud-cli/tests/issue_filters_smoke.py +++ b/fp-cloud-cli/tests/issue_filters_smoke.py @@ -34,18 +34,22 @@ def do_GET(self): self.calls.append({"path": parsed.path, "filters": filters}) route = parsed.path.removeprefix("/v1").removeprefix("/api") issue = {"id": "i1", "title": "Tool timeouts", "state": "firing", - "scope": {"harnesses": ["claude"], "agents": ["claude-repo"]}} + "harnesses": ["claude"], "agent_ids": ["claude-repo"], + "environments": ["prod"], "machine_ids": ["m1"], + "scope_computed_at": "2026-10-14T00:00:00Z"} status = 200 if route == "/issues/facets": status = 404 if self.legacy else 200 data = {"error": "not found"} if self.legacy else { "harnesses": [{"value": "claude", "count": 2}], "agents": [{"value": "claude-repo", "count": 2}], + "alerts": [{"id": "a1", "name": "Tool failure alert", "count": 2}], + "audits": [{"id": "au1", "name": "Weekly quality audit", "count": 1}], } elif route == "/issues/count": - data = {"count": 2} if self.legacy else {"count": 2, "filters": filters} + data = {"count": 0 if self.legacy else 2} elif route == "/issues": - data = [issue] if self.legacy else {"issues": [issue], "filters": filters} + data = [{"id": "old", "title": "Old issue"}] if self.legacy else [issue] else: status, data = 404, {"error": "unknown route"} raw = json.dumps(data).encode() @@ -76,6 +80,7 @@ def main(): mode = "key" if key_mode else "session" for command in ("list", "count", "facets"): for as_json in (False, True): + before = len(Handler.calls) result = subprocess.run([ args.fp, "--base-url", f"http://127.0.0.1:{server.server_port}", *auth, *(["--json"] if as_json else []), "issues", command, @@ -84,20 +89,36 @@ def main(): "--until", "2026-10-14T00:00:00Z", "--search", "tool timeout", ], env=env, text=True, capture_output=True, timeout=20) assert result.returncode == 0, result.stderr + assert len(Handler.calls) == before + 1, "normal calls must not probe facets" if as_json: data = json.loads(result.stdout) assert data["count"] == 2 if command == "count" else bool(data) + if command == "list": + row = data["issues"][0] + assert row["harnesses"] == ["claude"] + assert row["agent_ids"] == ["claude-repo"] + assert row["environments"] == ["prod"] + assert row["machine_ids"] == ["m1"] + assert row["scope_computed_at"] == "2026-10-14T00:00:00Z" + assert "scope" not in row + elif command == "facets": + assert data["alerts"][0]["id"] == "a1" + assert data["audits"][0]["id"] == "au1" elif command == "list": assert "[harness:claude]" in result.stdout assert "[agent:claude-repo]" in result.stdout elif command == "facets": assert "claude" in result.stdout and "2" in result.stdout + assert "Tool failure alert" in result.stdout + assert "Weekly quality audit" in result.stdout + assert "a1" not in result.stdout and "au1" not in result.stdout assert "unfiltered" not in result.stderr stem = f"issues-{mode}-{command}-{'json' if as_json else 'human'}" (artifacts / f"{stem}.txt").write_text(result.stdout + result.stderr) assert Handler.calls[-1]["filters"]["harness"] == "claude,codex" Handler.legacy = True for command in ("list", "count", "facets"): + before = len(Handler.calls) result = subprocess.run([ args.fp, "--base-url", f"http://127.0.0.1:{server.server_port}", *auth, "--json", "issues", command, "--harness", "claude", @@ -109,6 +130,7 @@ def main(): else: assert result.returncode == 0 assert result.stderr.count("unfiltered") == 1 + assert len(Handler.calls) == before + (2 if command == "count" else 1) (artifacts / f"issues-{mode}-{command}-legacy.txt").write_text( result.stdout + result.stderr) Handler.legacy = False diff --git a/fp-cloud-cli/tests/test_issue_filters.py b/fp-cloud-cli/tests/test_issue_filters.py index f13441e46..0cac4738b 100644 --- a/fp-cloud-cli/tests/test_issue_filters.py +++ b/fp-cloud-cli/tests/test_issue_filters.py @@ -37,8 +37,26 @@ "opened_to": "2026-10-14T00:00:00Z", "q": "tool failures & latency", } -FACETS = {"harnesses": [{"value": "claude", "count": 3}], - "agents": [{"value": "claude-repo", "count": 2}]} +ISSUE = { + "id": "i1", "title": "Timeouts", + "agent_ids": ["claude-repo"], "harnesses": ["claude"], + "environments": ["prod"], "machine_ids": ["m1"], + "scope_computed_at": "2026-10-14T00:00:00Z", +} +FACETS = { + "harnesses": [{"value": "claude", "count": 3}], + "agents": [{"value": "claude-repo", "count": 2}], + "alerts": [{"id": "a1", "name": "Tool failure alert", "count": 4}], + "audits": [{"id": "au1", "name": "Weekly quality audit", "count": 5}], +} + + +@pytest.fixture(autouse=True) +def fresh_filter_detection(monkeypatch): + """Each test is a new process; individual tests can exercise repeated calls.""" + monkeypatch.setattr(client, "_issue_facets_probed", False) + monkeypatch.setattr(client, "_issue_facets_available", None) + monkeypatch.setattr(client, "_issue_filters_warned", False) @pytest.mark.parametrize("key_mode", [False, True]) @@ -47,9 +65,8 @@ def test_all_filters_in_session_and_key_mode(logged_in, runner, key_mode, command): prefix = "/v1" if key_mode else "/api" path = f"{BASE}{prefix}/issues" + (f"/{command}" if command != "list" else "") - response = {"issues": [{"id": "i1", "scope": {"harnesses": ["claude"], "agents": ["claude-repo"]}}], - "filters": FILTERS} if command == "list" else ( - {"count": 3, "filters": FILTERS} if command == "count" else FACETS) + response = [ISSUE] if command == "list" else ( + {"count": 3} if command == "count" else FACETS) route = respx.get(path).respond(200, json=response) auth = ["--api-key", "key"] if key_mode else [] result = runner.invoke(app, ["--json", *auth, "issues", command, @@ -68,31 +85,46 @@ def test_all_filters_in_session_and_key_mode(logged_in, runner, key_mode, comman assert "authorization" not in request.headers payload = json.loads(result.stdout) if command == "list": - assert payload["issues"][0]["scope"]["harnesses"] == ["claude"] + row = payload["issues"][0] + for key in ("agent_ids", "harnesses", "environments", "machine_ids", "scope_computed_at"): + assert row[key] == ISSUE[key] + assert "scope" not in row elif command == "count": assert payload == {"count": 3} else: assert payload == FACETS - assert len(respx.calls) == 1, "echoed filters need no capability probe" + assert len(respx.calls) == 1, "normal filtered calls need no capability probe" + + +@pytest.mark.parametrize("key_mode", [False, True]) +@respx.mock +def test_old_nonempty_list_warns_without_extra_request(logged_in, runner, key_mode): + prefix = "/v1" if key_mode else "/api" + respx.get(f"{BASE}{prefix}/issues").respond(200, json=[{"id": "old"}]) + auth = ["--api-key", "key"] if key_mode else [] + result = runner.invoke(app, ["--json", "--quiet", *auth, "issues", "list", + "--harness", "claude", "--env", "prod"]) + assert result.exit_code == 0, result.output + assert result.stderr.count("not available on this FailproofAI Cloud yet") == 1 + assert "unfiltered" in result.stderr + assert len(respx.calls) == 1 + assert json.loads(result.stdout)["issues"][0]["id"] == "old" @pytest.mark.parametrize("key_mode", [False, True]) @pytest.mark.parametrize("command", ["list", "count"]) @respx.mock -def test_old_cloud_warns_once_but_preserves_json(logged_in, runner, key_mode, command): +def test_old_empty_result_probes_once_and_preserves_json(logged_in, runner, key_mode, command): prefix = "/v1" if key_mode else "/api" path = f"{BASE}{prefix}/issues" + ("/count" if command == "count" else "") - respx.get(path).respond(200, json=[{"id": "old"}] if command == "list" else {"count": 8}) + respx.get(path).respond(200, json=[] if command == "list" else {"count": 0}) facets = respx.get(f"{BASE}{prefix}/issues/facets").respond(404, json={"error": "not found"}) auth = ["--api-key", "key"] if key_mode else [] - result = runner.invoke(app, ["--json", "--quiet", *auth, "issues", command, - "--harness", "claude", "--env", "prod"]) + result = runner.invoke(app, ["--json", "--quiet", *auth, "issues", command, "--harness", "claude"]) assert result.exit_code == 0, result.output assert result.stderr.count("not available on this FailproofAI Cloud yet") == 1 - assert "unfiltered" in result.stderr assert facets.call_count == 1 - payload = json.loads(result.stdout) - assert (payload["issues"][0]["id"] == "old") if command == "list" else payload == {"count": 8} + assert json.loads(result.stdout) == ({"issues": []} if command == "list" else {"count": 0}) @pytest.mark.parametrize("key_mode", [False, True]) @@ -108,8 +140,8 @@ def test_missing_facets_has_existing_availability_error(logged_in, runner, key_m @respx.mock -def test_bare_list_on_new_cloud_probes_facets_without_warning(logged_in, runner): - respx.get(f"{BASE}/api/issues").respond(200, json=[{"id": "i1"}]) +def test_empty_list_on_new_cloud_probes_facets_without_warning(logged_in, runner): + respx.get(f"{BASE}/api/issues").respond(200, json=[]) route = respx.get(f"{BASE}/api/issues/facets").respond(200, json=FACETS) result = runner.invoke(app, ["--json", "issues", "list", "--source", "manual"]) assert result.exit_code == 0, result.output @@ -118,19 +150,49 @@ def test_bare_list_on_new_cloud_probes_facets_without_warning(logged_in, runner) @respx.mock -def test_partial_echo_still_checks_capability(logged_in, runner): - respx.get(f"{BASE}/api/issues").respond(200, json={"issues": [], "filters": {"harness": "claude"}}) - route = respx.get(f"{BASE}/api/issues/facets").respond(404, json={}) - result = runner.invoke(app, ["--json", "issues", "list", "--harness", "claude", "--env", "prod"]) - assert result.exit_code == 0 +@pytest.mark.parametrize("rows", [ + [{"id": "manual", "harnesses": []}], + [{"id": "legacy"}, ISSUE], +]) +def test_any_raw_harnesses_key_confirms_support_without_probe(logged_in, runner, rows): + respx.get(f"{BASE}/api/issues").respond(200, json=rows) + result = runner.invoke(app, ["--json", "issues", "list", "--source", "manual"]) + assert result.exit_code == 0, result.output + assert len(respx.calls) == 1 + assert "unfiltered" not in result.stderr + + +@pytest.mark.parametrize("available", [True, False]) +@respx.mock +def test_empty_result_probe_and_warning_are_once_per_process(logged_in, runner, available): + respx.get(f"{BASE}/api/issues").respond(200, json=[]) + respx.get(f"{BASE}/api/issues/count").respond(200, json={"count": 0}) + route = respx.get(f"{BASE}/api/issues/facets").respond( + 200 if available else 404, json=FACETS if available else {}) + results = [ + runner.invoke(app, ["--json", "issues", command, "--harness", "claude"]) + for command in ("list", "count", "list") + ] + assert all(result.exit_code == 0 for result in results) assert route.call_count == 1 - assert "unfiltered" in result.stderr + warnings = "".join(result.stderr for result in results) + assert warnings.count("unfiltered") == (0 if available else 1) + + +@respx.mock +def test_legacy_row_warning_is_once_per_process(logged_in, runner): + respx.get(f"{BASE}/api/issues").respond(200, json=[{"id": "old"}]) + results = [runner.invoke(app, ["--json", "issues", "list", "--harness", "claude"]) + for _ in range(2)] + assert all(result.exit_code == 0 for result in results) + assert "".join(result.stderr for result in results).count("unfiltered") == 1 + assert len(respx.calls) == 2 @pytest.mark.parametrize("status", [401, 403, 500]) @respx.mock def test_probe_errors_are_not_disguised_as_older_cloud(logged_in, runner, status): - respx.get(f"{BASE}/api/issues/count").respond(200, json={"count": 8}) + respx.get(f"{BASE}/api/issues/count").respond(200, json={"count": 0}) respx.get(f"{BASE}/api/issues/facets").respond(status, json={"error": "probe failed"}) result = runner.invoke(app, ["--json", "issues", "count", "--harness", "claude"]) assert result.exit_code != 0 @@ -156,7 +218,7 @@ def test_relative_since_keeps_until_and_csv_is_normalized(logged_in, runner, mon monkeypatch.setattr(dates, "resolve_range", lambda *args: real(*args, now=now)) expected = {"harness": "claude,codex", "opened_from": "2026-10-07T00:00:00Z", "opened_to": "2026-10-14T00:00:00Z"} - route = respx.get(f"{BASE}/api/issues/count").respond(200, json={"count": 0, "applied_filters": expected}) + route = respx.get(f"{BASE}/api/issues/count").respond(200, json={"count": 3}) result = runner.invoke(app, ["--json", "issues", "count", "--harness", "claude, codex", "--harness", "claude", "--since", "7d", "--until", "2026-10-14T00:00:00Z"]) @@ -171,31 +233,47 @@ def test_existing_alert_id_path_is_kept_and_combined_filters_use_issues(logged_i result = runner.invoke(app, ["--json", "issues", "list", "--alert-id", "a1"]) assert result.exit_code == 0, result.output assert legacy.called - route = respx.get(f"{BASE}/api/issues").respond(200, json={"issues": [], "filters": {"harness": "codex"}}) + route = respx.get(f"{BASE}/api/issues").respond(200, json=[{**ISSUE, "harnesses": ["codex"]}]) result = runner.invoke(app, ["--json", "issues", "list", "--alert-id", "a1", "--harness", "codex"]) assert result.exit_code == 0, result.output assert dict(route.calls[0].request.url.params) == {"limit": "50", "harness": "codex", "alert_id": "a1"} def test_human_rows_include_harness_and_agent_tags(capsys): - output.render_incidents([Incident.from_dict({ - "id": "i1", "title": "Timeouts", - "scope": {"harnesses": ["claude"], "agents": ["claude-repo"]}, - })]) + output.render_incidents([Incident.from_dict(ISSUE)]) out = capsys.readouterr().out assert "[harness:claude]" in out assert "[agent:claude-repo]" in out +def test_older_issue_defaults_scope_fields_without_inventing_tags(capsys): + issue = Incident.from_dict({"id": "old", "title": "Old issue"}) + assert issue.agent_ids == issue.harnesses == issue.environments == issue.machine_ids == [] + assert issue.scope_computed_at is None + output.render_incidents([issue]) + out = capsys.readouterr().out + assert "[harness:" not in out and "[agent:" not in out + + def test_human_facets_show_values_counts_and_empty_state(capsys): output.render_issue_facets(FACETS) out = capsys.readouterr().out assert "claude" in out and "3" in out and "claude-repo" in out + assert "Tool failure alert" in out and "4" in out + assert "Weekly quality audit" in out and "5" in out + assert "a1" not in out and "au1" not in out assert "{'value'" not in out output.render_issue_facets({}) assert "no issue filter values" in capsys.readouterr().out +def test_unnamed_alert_and_audit_facets_fall_back_to_id(capsys): + output.render_issue_facets({"alerts": [{"id": "a1", "name": None, "count": 2}], + "audits": [{"id": "au1", "name": "", "count": 1}]}) + out = capsys.readouterr().out + assert "a1" in out and "au1" in out + + def test_programmatic_client_reports_missing_filters_without_printing(): def handle(request): return httpx.Response(404, json={}) if request.url.path.endswith("/facets") else httpx.Response(200, json=[]) From 6263daed0dae2dd11aed4d592a490e2c5addcef2 Mon Sep 17 00:00:00 2001 From: Chetan Raghuvanshi <145042127+chhhee10@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:54:07 +0000 Subject: [PATCH 6/6] fix(issues): keep agent filters and restore collector output to base --- .github/workflows/issue-filters.yml | 17 ---- CHANGELOG.md | 2 +- __tests__/ci/issue-filters-workflow.test.ts | 9 +- crates/fpai-collect/src/sources/hooks/mod.rs | 11 +-- .../src/sources/hooks/transform.rs | 4 +- crates/fpai-collect/src/spool.rs | 98 ------------------- crates/fpai-collect/src/uploader.rs | 16 +-- crates/fpai-collect/tests/hooks_source.rs | 54 ---------- crates/fpai-collect/tests/uploader.rs | 42 -------- fp-cloud-cli/CHANGELOG.md | 2 +- fp-cloud-cli/README.md | 11 +-- fp-cloud-cli/fp_cli/client.py | 4 +- .../fp_cli/commands/incidents_cmds.py | 13 +-- fp-cloud-cli/fp_cli/models.py | 2 - fp-cloud-cli/fp_cli/output.py | 9 +- fp-cloud-cli/skill/SKILL.md | 4 +- fp-cloud-cli/skill/references/commands.md | 8 +- fp-cloud-cli/tests/issue_filters_smoke.py | 11 +-- fp-cloud-cli/tests/test_issue_filters.py | 67 ++++++++----- 19 files changed, 82 insertions(+), 302 deletions(-) diff --git a/.github/workflows/issue-filters.yml b/.github/workflows/issue-filters.yml index d71af3394..0b0cc50fc 100644 --- a/.github/workflows/issue-filters.yml +++ b/.github/workflows/issue-filters.yml @@ -6,7 +6,6 @@ on: pull_request: branches: [feat/fp-cli-parity] paths: - - "crates/fpai-collect/**" - "fp-cloud-cli/**" - ".github/workflows/issue-filters.yml" - "CHANGELOG.md" @@ -19,22 +18,6 @@ concurrency: cancel-in-progress: true jobs: - collector: - name: Collector harness - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@stable - with: - toolchain: "1.97.1" - components: rustfmt, clippy - - run: cargo fmt --all -- --check - - run: cargo clippy -p fpai-collect --all-targets -- -D warnings - - run: cargo test -p fpai-collect - cloud-cli: name: Issue filters (Python ${{ matrix.python-version }}) runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d514ed74..6cf8dd2c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Features -- Stamp harness provenance on every collected event, using the source kind, the hook row's CLI, or sdk for SDK batches, without replacing existing values; add Cloud CLI issue filters and facets (#900). +- Add Cloud CLI issue filters and facets for agent, environment, machine, source/alert/audit, assignee, opened dates and title/summary text (#900). - Cloud CLI parity: hosted code, LLM-judge and Jev evaluator authoring/deployment, dashboards, sentiment, assistant artifacts/approvals, org controls and the shared mint/pink screen language (#885) diff --git a/__tests__/ci/issue-filters-workflow.test.ts b/__tests__/ci/issue-filters-workflow.test.ts index 69c7bcb22..278d7a927 100644 --- a/__tests__/ci/issue-filters-workflow.test.ts +++ b/__tests__/ci/issue-filters-workflow.test.ts @@ -15,15 +15,12 @@ describe("issue-filter stacked verification", () => { expect(workflow.on.pull_request.branches).toContain("feat/fp-cli-parity"); expect(workflow.on.workflow_dispatch).toBeUndefined(); expect(workflow.permissions).toEqual({ contents: "read" }); - expect(workflow.on.pull_request.paths).toContain("crates/fpai-collect/**"); expect(workflow.on.pull_request.paths).toContain("fp-cloud-cli/**"); }); - it("runs collector formatting, clippy and all cargo tests", () => { - const run = scripts(workflow.jobs.collector); - expect(run).toContain("cargo fmt --all -- --check"); - expect(run).toContain("cargo clippy -p fpai-collect --all-targets -- -D warnings"); - expect(run).toContain("cargo test -p fpai-collect"); + it("keeps this client-only feature's verification scoped to the CLI", () => { + expect(Object.keys(workflow.jobs)).toEqual(["cloud-cli"]); + expect(workflow.on.pull_request.paths).not.toContain("crates/fpai-collect/**"); }); it("tests both auth modes across supported Python versions and a clean wheel", () => { diff --git a/crates/fpai-collect/src/sources/hooks/mod.rs b/crates/fpai-collect/src/sources/hooks/mod.rs index be6c8abe0..53446f28c 100644 --- a/crates/fpai-collect/src/sources/hooks/mod.rs +++ b/crates/fpai-collect/src/sources/hooks/mod.rs @@ -256,7 +256,6 @@ async fn poll_once( .or_insert_with(|| AllowBucket { session_id: key.0.clone(), agent_id: transform::agent_id(&row), - integration: key.5.clone(), event_name: key.1.clone(), tool_name: key.2.clone(), minute_ms: key.3, @@ -274,10 +273,7 @@ async fn poll_once( } for event in transform::to_events(&row, this_offset, environment) { - writer - .push_with_harness(event, row.integration.as_deref().unwrap_or("hooks")) - .await - .map_err(io_err)?; + writer.push(event).await.map_err(io_err)?; emitted += 1; } } @@ -297,10 +293,7 @@ async fn poll_once( for bucket in buckets.values() { if let Some(event) = bucket.to_event(environment) { - writer - .push_with_harness(event, &bucket.integration) - .await - .map_err(io_err)?; + writer.push(event).await.map_err(io_err)?; emitted += 1; } } diff --git a/crates/fpai-collect/src/sources/hooks/transform.rs b/crates/fpai-collect/src/sources/hooks/transform.rs index a7e2da507..e3eb312a7 100644 --- a/crates/fpai-collect/src/sources/hooks/transform.rs +++ b/crates/fpai-collect/src/sources/hooks/transform.rs @@ -826,7 +826,6 @@ pub fn to_events(row: &HookRow, offset: u64, environment: &str) -> Vec { pub struct AllowBucket { pub session_id: String, pub agent_id: String, - pub integration: String, pub event_name: String, pub tool_name: Option, /// Truncated to the minute; also the bucket's emitted timestamp. @@ -853,7 +852,7 @@ pub struct AllowBucket { /// emitted is wrong for the rest, which is worse than no attribution: it moves /// a count into a bucket someone is using to judge a rollout. Splitting instead /// costs extra events only when a minute genuinely mixed sources. -pub type BucketKey = (String, String, Option, i64, Attribution, String); +pub type BucketKey = (String, String, Option, i64, Attribution); pub fn bucket_key(row: &HookRow) -> Option { let session = row.session_id.clone().filter(|s| !s.is_empty())?; @@ -865,7 +864,6 @@ pub fn bucket_key(row: &HookRow) -> Option { row.tool_name.clone(), minute, Attribution::of(row), - row.integration.clone().unwrap_or_else(|| "hooks".into()), )) } diff --git a/crates/fpai-collect/src/spool.rs b/crates/fpai-collect/src/spool.rs index 2d195a250..562774ace 100644 --- a/crates/fpai-collect/src/spool.rs +++ b/crates/fpai-collect/src/spool.rs @@ -44,18 +44,6 @@ const MIN_BATCH_BYTES: u64 = 64 * 1024; /// Default batch cap, matching the ingest endpoint's body limit. pub const DEFAULT_MAX_BATCH_BYTES: u64 = 8 * 1024 * 1024; -/// Fill provenance only when absent. SDK batches arrive already spooled, so -/// the uploader uses this same stamp without writing a second batch. -pub(crate) fn stamp_harness(event: &mut Value, harness: &str) -> bool { - if let Some(obj) = event.as_object_mut() - && !obj.contains_key("harness") - { - obj.insert("harness".into(), Value::String(harness.into())); - return true; - } - false -} - /// Accumulates events and writes them out as size-bounded batches. pub struct SpoolWriter { dir: PathBuf, @@ -183,8 +171,6 @@ impl SpoolWriter { // disk. self.redacted += crate::redact::scrub_value(&mut event, self.redact) as u64; - stamp_harness(&mut event, &self.prefix); - // Stamp the machine id here, at the one point every event passes // through — a source cannot forget it and a new source inherits it. Set // only when absent, so an event that already names its machine (a @@ -259,17 +245,6 @@ impl SpoolWriter { Ok(()) } - /// Hook batches contain multiple CLIs. Use the row's CLI, not its agent id. - pub async fn push_with_harness( - &mut self, - mut event: Value, - harness: &str, - ) -> std::io::Result<()> { - stamp_harness(&mut event, harness); - // The later stamp_harness(prefix) in push is a no-op: the key is already set. - self.push(event).await - } - /// Durably write the buffered events as one batch. No-op when empty. /// /// Callers MUST await this before advancing any persisted cursor. The @@ -388,79 +363,6 @@ mod tests { v } - #[tokio::test] - async fn every_source_stamps_harness_without_guessing_from_the_agent_id() { - for source in [ - "claude", - "codex", - "copilot", - "cursor", - "pi", - "factory", - "antigravity", - "goose", - "opencode", - "hermes", - "devin", - "openclaw", - "sdk", - ] { - let dir = tmpdir(source); - let mut w = SpoolWriter::new(dir.clone(), DEFAULT_MAX_BATCH_BYTES, source, "s") - .with_label(Some("extra-path".into())); - for kind in ["agent_start", "tool_use", "agent_end"] { - w.push(json!({"type": kind, "agent_id": "not-a-harness"})) - .await - .unwrap(); - } - w.flush().await.unwrap(); - let body = std::fs::read_to_string(dir.join(batches(&dir).remove(0))).unwrap(); - for line in body.lines() { - let event: Value = serde_json::from_str(line).unwrap(); - assert_eq!(event["harness"], source); - } - std::fs::remove_dir_all(&dir).ok(); - } - } - - #[tokio::test] - async fn hooks_use_the_rows_cli_and_existing_harness_is_untouched() { - let dir = tmpdir("harness-hooks"); - let mut w = SpoolWriter::new(dir.clone(), DEFAULT_MAX_BATCH_BYTES, "hooks", "s"); - for cli in ["claude", "codex", "hermes"] { - w.push_with_harness(json!({"type": "hook_completed"}), cli) - .await - .unwrap(); - } - for harness in [json!("original"), Value::Null, json!("")] { - w.push(json!({"harness": harness})).await.unwrap(); - w.push_with_harness(json!({"harness": harness}), "copilot") - .await - .unwrap(); - } - w.flush().await.unwrap(); - let body = std::fs::read_to_string(dir.join(batches(&dir).remove(0))).unwrap(); - let values: Vec = body - .lines() - .map(|line| serde_json::from_str::(line).unwrap()["harness"].clone()) - .collect(); - assert_eq!( - values, - vec![ - json!("claude"), - json!("codex"), - json!("hermes"), - json!("original"), - json!("original"), - Value::Null, - Value::Null, - json!(""), - json!("") - ] - ); - std::fs::remove_dir_all(&dir).ok(); - } - #[tokio::test] async fn writes_one_batch_and_leaves_no_tmp_behind() { let dir = tmpdir("basic"); diff --git a/crates/fpai-collect/src/uploader.rs b/crates/fpai-collect/src/uploader.rs index dccb94536..ee527d9a8 100644 --- a/crates/fpai-collect/src/uploader.rs +++ b/crates/fpai-collect/src/uploader.rs @@ -268,14 +268,7 @@ impl Uploader { Err(e) => return Err(UploadError::Io(e)), }; - // Both SDKs (including the legacy spool) publish `event-*.jsonl`. - // The name survives parking/retries, so provenance is stable even - // after the batch moves out of custom-agents/events into failed/. - let sdk = path - .file_name() - .and_then(|s| s.to_str()) - .is_some_and(|name| name.starts_with("event-")); - let bytes = prepare_batch(&bytes, self.redact, sdk); + let bytes = redact_batch(&bytes, self.redact); for chunk in split_lines(&bytes, self.max_upload_bytes) { self.post_batch(path, chunk).await?; } @@ -705,8 +698,8 @@ fn lower_hex(bytes: &[u8]) -> String { out } -fn prepare_batch(bytes: &[u8], mode: Redact, sdk: bool) -> Vec { - if mode == Redact::Off && !sdk { +fn redact_batch(bytes: &[u8], mode: Redact) -> Vec { + if mode == Redact::Off { return bytes.to_vec(); } @@ -717,8 +710,7 @@ fn prepare_batch(bytes: &[u8], mode: Redact, sdk: bool) -> Vec { .map_or((line, false), |body| (body, true)); match serde_json::from_slice::(body) { Ok(mut event) => { - let stamped = sdk && crate::spool::stamp_harness(&mut event, "sdk"); - if crate::redact::scrub_value(&mut event, mode) > 0 || stamped { + if crate::redact::scrub_value(&mut event, mode) > 0 { event .serialize(&mut serde_json::Serializer::new(&mut out)) .expect("serializing JSON into Vec cannot fail"); diff --git a/crates/fpai-collect/tests/hooks_source.rs b/crates/fpai-collect/tests/hooks_source.rs index 8e939aca2..5c94ab06e 100644 --- a/crates/fpai-collect/tests/hooks_source.rs +++ b/crates/fpai-collect/tests/hooks_source.rs @@ -115,60 +115,6 @@ fn epoch_millis_become_rfc3339_with_microseconds() { assert!(s < s1); } -#[tokio::test] -async fn hooks_stamp_each_cli_on_pairs_and_separate_allow_buckets() { - for verbosity in [HooksVerbosity::All, HooksVerbosity::Decisions] { - let store = tmpdir("harness-store"); - let state = tmpdir("harness-state"); - let spool = tmpdir("harness-spool"); - let mut rows = String::new(); - for cli in [ - "claude", - "codex", - "copilot", - "cursor", - "pi", - "factory", - "antigravity", - "goose", - "opencode", - "hermes", - "devin", - "openclaw", - ] { - for decision in ["allow", "deny"] { - let mut row: Value = - serde_json::from_str(&allow_row(1785740912184, "Bash")).unwrap(); - row["integration"] = serde_json::json!(cli); - row["decision"] = serde_json::json!(decision); - rows.push_str(&row.to_string()); - rows.push('\n'); - } - } - fs::write(store.join("page.jsonl"), rows).unwrap(); - run_once(&store, &state, &spool, verbosity).await; - let events = spooled(&spool); - assert_eq!( - events.len(), - if verbosity == HooksVerbosity::All { - 48 - } else { - 36 - } - ); - for event in events { - let cli = event["harness"].as_str().unwrap(); - assert_eq!(event["agent_id"], format!("{cli}-repo")); - if verbosity == HooksVerbosity::Decisions && event["outcome"] == "allow" { - assert_eq!(event["failproofai_allow_count"], 1); - } - } - fs::remove_dir_all(&store).ok(); - fs::remove_dir_all(&state).ok(); - fs::remove_dir_all(&spool).ok(); - } -} - #[test] fn a_leap_day_is_handled() { // The reason this uses a date library rather than hand-rolled maths. diff --git a/crates/fpai-collect/tests/uploader.rs b/crates/fpai-collect/tests/uploader.rs index 7a7f5730b..68cb1cb6c 100644 --- a/crates/fpai-collect/tests/uploader.rs +++ b/crates/fpai-collect/tests/uploader.rs @@ -88,48 +88,6 @@ async fn a_2xx_with_an_accepting_ack_deletes_the_batch() { fs::remove_dir_all(&failed).ok(); } -#[tokio::test] -async fn sdk_harness_is_stamped_even_without_redaction_and_survives_retry() { - for mode in [Redact::Minimal, Redact::Off] { - let server = MockServer::start().await; - Mock::given(method("POST")) - .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ - "accepted": 3, "skipped": 0 - }))) - .mount(&server) - .await; - let spool = tmpdir("sdk-harness"); - let failed = tmpdir("sdk-harness-failed"); - // Parked SDK files retain their event-* stem. - let batch = spool.join("event-s-1-0.a1.jsonl"); - fs::write( - &batch, - concat!( - "{\"type\":\"agent_start\",\"agent_id\":\"claude-custom\"}\n", - "{\"type\":\"tool_use\",\"harness\":\"original\"}\n", - "{\"type\":\"agent_end\",\"harness\":null}\n", - ), - ) - .unwrap(); - uploader(&server, &failed) - .with_redact(mode) - .upload_file(&batch) - .await - .unwrap(); - let requests = server.received_requests().await.unwrap(); - let body = String::from_utf8(requests[0].body.clone()).unwrap(); - let events: Vec = body - .lines() - .map(|line| serde_json::from_str(line).unwrap()) - .collect(); - assert_eq!(events[0]["harness"], "sdk"); - assert_eq!(events[1]["harness"], "original"); - assert!(events[2]["harness"].is_null()); - fs::remove_dir_all(&spool).ok(); - fs::remove_dir_all(&failed).ok(); - } -} - #[tokio::test] async fn sdk_batches_are_redacted_before_upload() { let server = MockServer::start().await; diff --git a/fp-cloud-cli/CHANGELOG.md b/fp-cloud-cli/CHANGELOG.md index bb92d9870..e450b69df 100644 --- a/fp-cloud-cli/CHANGELOG.md +++ b/fp-cloud-cli/CHANGELOG.md @@ -4,7 +4,7 @@ ### Added -- Filter issues list/count and discover facets by agent, harness, environment, machine, source/alert/audit, assignee, opened dates and title/summary text in session or key mode; show scope tags and warn when an older Cloud may ignore filters (#900). +- Filter issues list/count and discover facets by agent, environment, machine, source/alert/audit, assignee, opened dates and title/summary text in session or key mode; show agent tags and warn when an older Cloud may ignore filters (#900). - Hosted evaluator authoring for code, LLM judge and Jev: create/deploy, AI draft specs, dry-run, labels, toggles, instances, schema, jobs and code-only backfill. - Dashboards and tiles; session overview/export/history/re-evaluate; latency and sentiment; query composition/export; alert recipients/issues and agent contracts. diff --git a/fp-cloud-cli/README.md b/fp-cloud-cli/README.md index 70483248e..fdd985093 100644 --- a/fp-cloud-cli/README.md +++ b/fp-cloud-cli/README.md @@ -133,28 +133,27 @@ Issue filters work with either a signed-in session or an API key carrying `issue ```bash fp --json issues facets --env prod -fp issues list --harness claude --harness codex --env prod --assignee me --since 7d +fp issues list --agent claude-repo --agent codex-repo --env prod --assignee me --since 7d fp --json issues count --source audit --audit weekly-audit --machine ci-runner-01 fp issues list --agent claude-checkout --assignee unassigned --search "tool timeout" \ --since 2026-10-01T00:00:00Z --until 2026-10-14T00:00:00Z ``` -`issues list`, `count` and `facets` share `--agent`, `--harness`, `--env`, +`issues list`, `count` and `facets` share `--agent`, `--env`, `--machine`, `--source alert|audit|manual`, `--alert`, `--audit` and `--assignee EMAIL|me|unassigned`. These accept repeated flags or comma-separated values (OR within a filter, AND between filters). `--since` accepts the existing `all|15m|1h|6h|24h|7d` presets or RFC3339; `--until` accepts RFC3339. Dates apply to issue opening, and `--search` searches title + summary. Human rows show the -stored harness/agent scope; facets show available values and counts. +stored agent scope; facets show available values and counts. Older Clouds may ignore the new query parameters. A returned issue's top-level -`harnesses` key confirms filter support without another request. Non-empty +`agent_ids` key confirms filter support without another request. Non-empty legacy rows warn once; empty lists or zero counts probe facets at most once per process, and a missing route warns once that results may be unfiltered. Positive counts make no extra request. Warnings leave stdout JSON usable. Calling missing `facets` directly exits 6 with “not available on this FailproofAI Cloud yet”. -Upgrade Cloud for filtering; the collector's new `harness` event field is -additive and ignored by old servers. Existing `list --alert-id` remains supported. +Upgrade Cloud for filtering. Existing `list --alert-id` remains supported. ### Cloud-managed policies diff --git a/fp-cloud-cli/fp_cli/client.py b/fp-cloud-cli/fp_cli/client.py index d2ff81a5a..6a18ffc8b 100644 --- a/fp-cloud-cli/fp_cli/client.py +++ b/fp-cloud-cli/fp_cli/client.py @@ -1356,7 +1356,7 @@ def _check_issue_filters( ctx: ClientContext, data: Any, filters: Optional[Dict[str, str]], on_unavailable: Optional[Callable[[], None]], ) -> None: - """New Cloud always sends `harnesses`, even when its array is empty. + """New Cloud always sends `agent_ids`, even when its array is empty. Non-empty lists confirm support from the raw rows (before model defaults fill missing keys), or warn for legacy rows without making another request. @@ -1368,7 +1368,7 @@ def _check_issue_filters( if not filters: return if isinstance(data, list) and data: - if any(isinstance(row, dict) and "harnesses" in row for row in data): + if any(isinstance(row, dict) and "agent_ids" in row for row in data): return unavailable = True else: diff --git a/fp-cloud-cli/fp_cli/commands/incidents_cmds.py b/fp-cloud-cli/fp_cli/commands/incidents_cmds.py index 3ade01db6..e3ec0609e 100644 --- a/fp-cloud-cli/fp_cli/commands/incidents_cmds.py +++ b/fp-cloud-cli/fp_cli/commands/incidents_cmds.py @@ -70,13 +70,13 @@ def _fail(state: AppState, exc: Exception, *, incident_id: str = "") -> None: def _issue_filters( - agent, harness, environment, machine, source, alert, audit, assignee, + agent, environment, machine, source, alert, audit, assignee, since, until, search, ) -> dict: """The shared list/count/facets contract; repeated values become CSV.""" filters = {} for key, values in ( - ("agent", agent), ("harness", harness), ("environment", environment), + ("agent", agent), ("environment", environment), ("machine", machine), ("source", source), ("alert_id", alert), ("audit_id", audit), ("assignee", assignee), ): @@ -124,7 +124,6 @@ def incidents_list( limit: int = typer.Option(50, "--limit", "-n", help="Max incidents to return."), show_id: bool = typer.Option(False, "--show-id", help="Show the full incident id instead of the short form (always full in --json)."), agent: Optional[List[str]] = typer.Option(None, "--agent", help="Agent id (repeatable or CSV)."), - harness: Optional[List[str]] = typer.Option(None, "--harness", help="Harness (repeatable or CSV)."), environment: Optional[List[str]] = typer.Option(None, "--env", help="Environment (repeatable or CSV)."), machine: Optional[List[str]] = typer.Option(None, "--machine", help="Machine id (repeatable or CSV)."), source: Optional[List[str]] = typer.Option(None, "--source", help="Source: alert, audit, manual (repeatable or CSV)."), @@ -153,7 +152,7 @@ def incidents_list( state: AppState = ctx.obj _validate_states(state_filter) validate_limit(limit) - filters = _issue_filters(agent, harness, environment, machine, source, alert, audit, assignee, since, until, search) + filters = _issue_filters(agent, environment, machine, source, alert, audit, assignee, since, until, search) if alert_id and "alert_id" in filters: raise typer.BadParameter("use --alert or --alert-id, not both") cctx = require_auth(state) @@ -172,7 +171,6 @@ def incidents_count( ctx: typer.Context, state_filter: Optional[str] = typer.Option(None, "--state", help="Filter by state(s) (CSV)."), agent: Optional[List[str]] = typer.Option(None, "--agent", help="Agent id (repeatable or CSV)."), - harness: Optional[List[str]] = typer.Option(None, "--harness", help="Harness (repeatable or CSV)."), environment: Optional[List[str]] = typer.Option(None, "--env", help="Environment (repeatable or CSV)."), machine: Optional[List[str]] = typer.Option(None, "--machine", help="Machine id (repeatable or CSV)."), source: Optional[List[str]] = typer.Option(None, "--source", help="Source: alert, audit, manual (repeatable or CSV)."), @@ -194,7 +192,7 @@ def incidents_count( """ state: AppState = ctx.obj _validate_states(state_filter) - filters = _issue_filters(agent, harness, environment, machine, source, alert, audit, assignee, since, until, search) + filters = _issue_filters(agent, environment, machine, source, alert, audit, assignee, since, until, search) cctx = require_auth(state) count = api.count_incidents(cctx, state=state_filter, filters=filters, on_filters_unavailable=_filters_unavailable) if state.json: @@ -207,7 +205,6 @@ def incidents_facets( ctx: typer.Context, state_filter: Optional[str] = typer.Option(None, "--state", help="Filter by state(s) (CSV)."), agent: Optional[List[str]] = typer.Option(None, "--agent", help="Agent id (repeatable or CSV)."), - harness: Optional[List[str]] = typer.Option(None, "--harness", help="Harness (repeatable or CSV)."), environment: Optional[List[str]] = typer.Option(None, "--env", help="Environment (repeatable or CSV)."), machine: Optional[List[str]] = typer.Option(None, "--machine", help="Machine id (repeatable or CSV)."), source: Optional[List[str]] = typer.Option(None, "--source", help="Source: alert, audit, manual (repeatable or CSV)."), @@ -226,7 +223,7 @@ def incidents_facets( """ state: AppState = ctx.obj _validate_states(state_filter) - filters = _issue_filters(agent, harness, environment, machine, source, alert, audit, assignee, since, until, search) + filters = _issue_filters(agent, environment, machine, source, alert, audit, assignee, since, until, search) data = api.issue_facets(require_auth(state), state=state_filter, filters=filters) if state.json: output.emit_json(data) diff --git a/fp-cloud-cli/fp_cli/models.py b/fp-cloud-cli/fp_cli/models.py index 0e0e75dcd..5e6682164 100644 --- a/fp-cloud-cli/fp_cli/models.py +++ b/fp-cloud-cli/fp_cli/models.py @@ -432,7 +432,6 @@ class Incident: activity: Optional[List[Dict[str, Any]]] = None #: Stored evidence scope, returned as top-level fields by Cloud. agent_ids: List[str] = field(default_factory=list) - harnesses: List[str] = field(default_factory=list) environments: List[str] = field(default_factory=list) machine_ids: List[str] = field(default_factory=list) scope_computed_at: Optional[str] = None @@ -468,7 +467,6 @@ def from_dict(cls, d: Dict[str, Any]) -> "Incident": comments=d.get("comments"), activity=d.get("activity"), agent_ids=list(d.get("agent_ids") or []), - harnesses=list(d.get("harnesses") or []), environments=list(d.get("environments") or []), machine_ids=list(d.get("machine_ids") or []), scope_computed_at=d.get("scope_computed_at"), diff --git a/fp-cloud-cli/fp_cli/output.py b/fp-cloud-cli/fp_cli/output.py index da47db2b9..edeac7b94 100644 --- a/fp-cloud-cli/fp_cli/output.py +++ b/fp-cloud-cli/fp_cli/output.py @@ -4384,9 +4384,8 @@ def render_incidents(incidents: Sequence[Any], *, show_id: bool = False) -> None identity = Text(label, style=theme.TEXT) # Keep tags with the title so narrow terminals never lose provenance # by dropping a rightmost column. - for key, tag in (("harnesses", "harness"), ("agent_ids", "agent")): - for value in getattr(i, key, None) or []: - identity.append(f" [{tag}:{value}]", style=theme.TEXT_DIM) + for value in getattr(i, "agent_ids", None) or []: + identity.append(f" [agent:{value}]", style=theme.TEXT_DIM) rows.append([ Text(iid, style=theme.TEXT_DIM), identity, @@ -4408,7 +4407,9 @@ def render_issue_facets(data: Dict[str, Any]) -> None: facets = data.get("facets", data) rows = [] for facet, values in facets.items(): - if not isinstance(values, list): + if facet not in ( + "agents", "environments", "machines", "sources", "alerts", "audits", "assignees", + ) or not isinstance(values, list): continue for item in values: if isinstance(item, dict): diff --git a/fp-cloud-cli/skill/SKILL.md b/fp-cloud-cli/skill/SKILL.md index 882c738e5..26d29bf0f 100644 --- a/fp-cloud-cli/skill/SKILL.md +++ b/fp-cloud-cli/skill/SKILL.md @@ -187,10 +187,10 @@ you need a flag you don't already know. - `errors` — errored events; `--aggregate` for count / sessions / agents / last-seen. - `usage` — current org usage for its fixed 30-day metering window; needs `usage:read`. - `list ` — **discover valid filter values first**: `envs agents event_types score_filters models hooks tools error_types`. -- `issues list|count|facets` — scope the failures inbox by agent/harness/environment/machine, +- `issues list|count|facets` — scope the failures inbox by agent/environment/machine, source or a specific alert/audit, assignee, opened dates and title/summary text. Use `fp --json issues facets --env prod` to discover values/counts, then - `fp --json issues list --env prod --harness claude --assignee me --since 7d`. + `fp --json issues list --env prod --agent claude-repo --assignee me --since 7d`. Full flags are in `references/commands.md`. If stderr warns that Cloud may have returned unfiltered results, do not treat the response as proof of the requested scope. diff --git a/fp-cloud-cli/skill/references/commands.md b/fp-cloud-cli/skill/references/commands.md index 37989bc1b..b25fdda23 100644 --- a/fp-cloud-cli/skill/references/commands.md +++ b/fp-cloud-cli/skill/references/commands.md @@ -188,18 +188,18 @@ The single board for everything needing human attention — alert breaches (`sou - `issues facets` — available filter values with counts; read-only, needs `issues:read`. `list`, `count` and `facets` share filters (session and API-key mode): -`--agent`, `--harness`, `--env`, `--machine`, `--source alert|audit|manual`, +`--agent`, `--env`, `--machine`, `--source alert|audit|manual`, `--alert `, `--audit `, `--assignee EMAIL|me|unassigned`. Each accepts repeated flags or comma-separated values. Different dimensions combine with AND; values within one dimension combine with OR. `--since` accepts the existing `all|15m|1h|6h|24h|7d` presets or an RFC3339 timestamp; `--until` accepts RFC3339. Dates filter when the issue **opened**, not when it last fired. `--search TEXT` -searches title + summary. Human issue rows read harness/agent tags from top-level -`harnesses`/`agent_ids`; `--json` also keeps `environments`, `machine_ids` and +searches title + summary. Human issue rows read agent tags from top-level +`agent_ids`; `--json` also keeps `environments`, `machine_ids` and `scope_computed_at`. Facets render alert/audit names; JSON retains their ids. Global `--json` goes before `issues`, and facets JSON preserves the server response. On an older Cloud, missing facets exits 6 with “not available on this FailproofAI Cloud -yet”. A returned row's `harnesses` key confirms list support without another request. +yet”. A returned row's `agent_ids` key confirms list support without another request. Non-empty legacy rows warn once. Empty lists/zero counts probe facets at most once per process and warn once if missing (also under `--json`/`--quiet`); positive counts make no extra request. diff --git a/fp-cloud-cli/tests/issue_filters_smoke.py b/fp-cloud-cli/tests/issue_filters_smoke.py index 1cac68d80..484766c06 100644 --- a/fp-cloud-cli/tests/issue_filters_smoke.py +++ b/fp-cloud-cli/tests/issue_filters_smoke.py @@ -34,14 +34,13 @@ def do_GET(self): self.calls.append({"path": parsed.path, "filters": filters}) route = parsed.path.removeprefix("/v1").removeprefix("/api") issue = {"id": "i1", "title": "Tool timeouts", "state": "firing", - "harnesses": ["claude"], "agent_ids": ["claude-repo"], + "agent_ids": ["claude-repo"], "environments": ["prod"], "machine_ids": ["m1"], "scope_computed_at": "2026-10-14T00:00:00Z"} status = 200 if route == "/issues/facets": status = 404 if self.legacy else 200 data = {"error": "not found"} if self.legacy else { - "harnesses": [{"value": "claude", "count": 2}], "agents": [{"value": "claude-repo", "count": 2}], "alerts": [{"id": "a1", "name": "Tool failure alert", "count": 2}], "audits": [{"id": "au1", "name": "Weekly quality audit", "count": 1}], @@ -84,7 +83,7 @@ def main(): result = subprocess.run([ args.fp, "--base-url", f"http://127.0.0.1:{server.server_port}", *auth, *(["--json"] if as_json else []), "issues", command, - "--harness", "claude", "--harness", "codex", "--env", "prod", + "--agent", "claude-repo", "--agent", "codex-repo", "--env", "prod", "--assignee", "unassigned", "--since", "2026-10-01T00:00:00Z", "--until", "2026-10-14T00:00:00Z", "--search", "tool timeout", ], env=env, text=True, capture_output=True, timeout=20) @@ -95,7 +94,6 @@ def main(): assert data["count"] == 2 if command == "count" else bool(data) if command == "list": row = data["issues"][0] - assert row["harnesses"] == ["claude"] assert row["agent_ids"] == ["claude-repo"] assert row["environments"] == ["prod"] assert row["machine_ids"] == ["m1"] @@ -105,7 +103,6 @@ def main(): assert data["alerts"][0]["id"] == "a1" assert data["audits"][0]["id"] == "au1" elif command == "list": - assert "[harness:claude]" in result.stdout assert "[agent:claude-repo]" in result.stdout elif command == "facets": assert "claude" in result.stdout and "2" in result.stdout @@ -115,13 +112,13 @@ def main(): assert "unfiltered" not in result.stderr stem = f"issues-{mode}-{command}-{'json' if as_json else 'human'}" (artifacts / f"{stem}.txt").write_text(result.stdout + result.stderr) - assert Handler.calls[-1]["filters"]["harness"] == "claude,codex" + assert Handler.calls[-1]["filters"]["agent"] == "claude-repo,codex-repo" Handler.legacy = True for command in ("list", "count", "facets"): before = len(Handler.calls) result = subprocess.run([ args.fp, "--base-url", f"http://127.0.0.1:{server.server_port}", - *auth, "--json", "issues", command, "--harness", "claude", + *auth, "--json", "issues", command, "--agent", "claude-repo", ], env=env, text=True, capture_output=True, timeout=20) json.loads(result.stdout) if command == "facets": diff --git a/fp-cloud-cli/tests/test_issue_filters.py b/fp-cloud-cli/tests/test_issue_filters.py index 0cac4738b..979c1112a 100644 --- a/fp-cloud-cli/tests/test_issue_filters.py +++ b/fp-cloud-cli/tests/test_issue_filters.py @@ -6,6 +6,7 @@ import httpx import pytest import respx +from typer.main import get_command from fp_cli import client, dates, output from fp_cli.app import app @@ -14,7 +15,6 @@ BASE = "http://dash.test" FILTER_FLAGS = [ "--agent", "claude-repo", "--agent", "codex-repo", - "--harness", "claude", "--harness", "codex", "--env", "prod", "--env", "staging", "--machine", "m1", "--machine", "m2", "--source", "alert", "--source", "audit", @@ -26,7 +26,6 @@ ] FILTERS = { "agent": "claude-repo,codex-repo", - "harness": "claude,codex", "environment": "prod,staging", "machine": "m1,m2", "source": "alert,audit", @@ -39,12 +38,11 @@ } ISSUE = { "id": "i1", "title": "Timeouts", - "agent_ids": ["claude-repo"], "harnesses": ["claude"], + "agent_ids": ["claude-repo"], "environments": ["prod"], "machine_ids": ["m1"], "scope_computed_at": "2026-10-14T00:00:00Z", } FACETS = { - "harnesses": [{"value": "claude", "count": 3}], "agents": [{"value": "claude-repo", "count": 2}], "alerts": [{"id": "a1", "name": "Tool failure alert", "count": 4}], "audits": [{"id": "au1", "name": "Weekly quality audit", "count": 5}], @@ -86,7 +84,7 @@ def test_all_filters_in_session_and_key_mode(logged_in, runner, key_mode, comman payload = json.loads(result.stdout) if command == "list": row = payload["issues"][0] - for key in ("agent_ids", "harnesses", "environments", "machine_ids", "scope_computed_at"): + for key in ("agent_ids", "environments", "machine_ids", "scope_computed_at"): assert row[key] == ISSUE[key] assert "scope" not in row elif command == "count": @@ -103,7 +101,7 @@ def test_old_nonempty_list_warns_without_extra_request(logged_in, runner, key_mo respx.get(f"{BASE}{prefix}/issues").respond(200, json=[{"id": "old"}]) auth = ["--api-key", "key"] if key_mode else [] result = runner.invoke(app, ["--json", "--quiet", *auth, "issues", "list", - "--harness", "claude", "--env", "prod"]) + "--agent", "claude-repo", "--env", "prod"]) assert result.exit_code == 0, result.output assert result.stderr.count("not available on this FailproofAI Cloud yet") == 1 assert "unfiltered" in result.stderr @@ -120,7 +118,7 @@ def test_old_empty_result_probes_once_and_preserves_json(logged_in, runner, key_ respx.get(path).respond(200, json=[] if command == "list" else {"count": 0}) facets = respx.get(f"{BASE}{prefix}/issues/facets").respond(404, json={"error": "not found"}) auth = ["--api-key", "key"] if key_mode else [] - result = runner.invoke(app, ["--json", "--quiet", *auth, "issues", command, "--harness", "claude"]) + result = runner.invoke(app, ["--json", "--quiet", *auth, "issues", command, "--agent", "claude-repo"]) assert result.exit_code == 0, result.output assert result.stderr.count("not available on this FailproofAI Cloud yet") == 1 assert facets.call_count == 1 @@ -151,10 +149,10 @@ def test_empty_list_on_new_cloud_probes_facets_without_warning(logged_in, runner @respx.mock @pytest.mark.parametrize("rows", [ - [{"id": "manual", "harnesses": []}], + [{"id": "manual", "agent_ids": []}], [{"id": "legacy"}, ISSUE], ]) -def test_any_raw_harnesses_key_confirms_support_without_probe(logged_in, runner, rows): +def test_any_raw_agent_ids_key_confirms_support_without_probe(logged_in, runner, rows): respx.get(f"{BASE}/api/issues").respond(200, json=rows) result = runner.invoke(app, ["--json", "issues", "list", "--source", "manual"]) assert result.exit_code == 0, result.output @@ -170,7 +168,7 @@ def test_empty_result_probe_and_warning_are_once_per_process(logged_in, runner, route = respx.get(f"{BASE}/api/issues/facets").respond( 200 if available else 404, json=FACETS if available else {}) results = [ - runner.invoke(app, ["--json", "issues", command, "--harness", "claude"]) + runner.invoke(app, ["--json", "issues", command, "--agent", "claude-repo"]) for command in ("list", "count", "list") ] assert all(result.exit_code == 0 for result in results) @@ -182,7 +180,7 @@ def test_empty_result_probe_and_warning_are_once_per_process(logged_in, runner, @respx.mock def test_legacy_row_warning_is_once_per_process(logged_in, runner): respx.get(f"{BASE}/api/issues").respond(200, json=[{"id": "old"}]) - results = [runner.invoke(app, ["--json", "issues", "list", "--harness", "claude"]) + results = [runner.invoke(app, ["--json", "issues", "list", "--agent", "claude-repo"]) for _ in range(2)] assert all(result.exit_code == 0 for result in results) assert "".join(result.stderr for result in results).count("unfiltered") == 1 @@ -194,14 +192,14 @@ def test_legacy_row_warning_is_once_per_process(logged_in, runner): def test_probe_errors_are_not_disguised_as_older_cloud(logged_in, runner, status): respx.get(f"{BASE}/api/issues/count").respond(200, json={"count": 0}) respx.get(f"{BASE}/api/issues/facets").respond(status, json={"error": "probe failed"}) - result = runner.invoke(app, ["--json", "issues", "count", "--harness", "claude"]) + result = runner.invoke(app, ["--json", "issues", "count", "--agent", "claude-repo"]) assert result.exit_code != 0 assert "unfiltered" not in result.stderr @pytest.mark.parametrize("command", ["list", "count", "facets"]) @pytest.mark.parametrize("flags", [ - ["--source", "invalid"], ["--harness", ""], ["--agent", "a,,b"], + ["--source", "invalid"], ["--agent", ""], ["--agent", "a,,b"], ["--since", "yesterday"], ["--until", "2026-10-14"], ]) @respx.mock @@ -216,11 +214,11 @@ def test_relative_since_keeps_until_and_csv_is_normalized(logged_in, runner, mon real = dates.resolve_range now = datetime(2026, 10, 14, tzinfo=timezone.utc) monkeypatch.setattr(dates, "resolve_range", lambda *args: real(*args, now=now)) - expected = {"harness": "claude,codex", "opened_from": "2026-10-07T00:00:00Z", + expected = {"agent": "claude-repo,codex-repo", "opened_from": "2026-10-07T00:00:00Z", "opened_to": "2026-10-14T00:00:00Z"} route = respx.get(f"{BASE}/api/issues/count").respond(200, json={"count": 3}) result = runner.invoke(app, ["--json", "issues", "count", - "--harness", "claude, codex", "--harness", "claude", + "--agent", "claude-repo, codex-repo", "--agent", "claude-repo", "--since", "7d", "--until", "2026-10-14T00:00:00Z"]) assert result.exit_code == 0, result.output assert dict(route.calls[0].request.url.params) == expected @@ -233,32 +231,44 @@ def test_existing_alert_id_path_is_kept_and_combined_filters_use_issues(logged_i result = runner.invoke(app, ["--json", "issues", "list", "--alert-id", "a1"]) assert result.exit_code == 0, result.output assert legacy.called - route = respx.get(f"{BASE}/api/issues").respond(200, json=[{**ISSUE, "harnesses": ["codex"]}]) - result = runner.invoke(app, ["--json", "issues", "list", "--alert-id", "a1", "--harness", "codex"]) + route = respx.get(f"{BASE}/api/issues").respond(200, json=[{**ISSUE, "agent_ids": ["codex-repo"]}]) + result = runner.invoke(app, ["--json", "issues", "list", "--alert-id", "a1", "--agent", "codex-repo"]) assert result.exit_code == 0, result.output - assert dict(route.calls[0].request.url.params) == {"limit": "50", "harness": "codex", "alert_id": "a1"} + assert dict(route.calls[0].request.url.params) == {"limit": "50", "agent": "codex-repo", "alert_id": "a1"} -def test_human_rows_include_harness_and_agent_tags(capsys): +def test_human_rows_include_agent_tags(capsys): output.render_incidents([Incident.from_dict(ISSUE)]) out = capsys.readouterr().out - assert "[harness:claude]" in out assert "[agent:claude-repo]" in out +@pytest.mark.parametrize("command", ["list", "count", "facets"]) +def test_issue_commands_offer_only_the_supported_filter_flags(command): + cmd = get_command(app).commands["issues"].commands[command] + flags = {flag for param in cmd.params for flag in getattr(param, "opts", [])} + expected = { + "--state", "--agent", "--env", "--machine", "--source", "--alert", "--audit", + "--assignee", "--since", "--until", "--search", + } + if command == "list": + expected.update(("--alert-id", "--limit", "-n", "--show-id")) + assert flags == expected + + def test_older_issue_defaults_scope_fields_without_inventing_tags(capsys): issue = Incident.from_dict({"id": "old", "title": "Old issue"}) - assert issue.agent_ids == issue.harnesses == issue.environments == issue.machine_ids == [] + assert issue.agent_ids == issue.environments == issue.machine_ids == [] assert issue.scope_computed_at is None output.render_incidents([issue]) out = capsys.readouterr().out - assert "[harness:" not in out and "[agent:" not in out + assert "[agent:" not in out def test_human_facets_show_values_counts_and_empty_state(capsys): output.render_issue_facets(FACETS) out = capsys.readouterr().out - assert "claude" in out and "3" in out and "claude-repo" in out + assert "2" in out and "claude-repo" in out assert "Tool failure alert" in out and "4" in out assert "Weekly quality audit" in out and "5" in out assert "a1" not in out and "au1" not in out @@ -274,11 +284,20 @@ def test_unnamed_alert_and_audit_facets_fall_back_to_id(capsys): assert "a1" in out and "au1" in out +def test_human_facets_ignore_dimensions_outside_the_issue_filters(capsys): + output.render_issue_facets({ + **FACETS, "unrelated": [{"value": "unsupported-value", "count": 1}], + }) + out = capsys.readouterr().out + assert "claude-repo" in out + assert "unrelated" not in out and "unsupported-value" not in out + + def test_programmatic_client_reports_missing_filters_without_printing(): def handle(request): return httpx.Response(404, json={}) if request.url.path.endswith("/facets") else httpx.Response(200, json=[]) ctx = client.ClientContext(BASE, token="tok", transport=httpx.MockTransport(handle)) warnings = [] - assert client.list_incidents(ctx, filters={"harness": "codex"}, + assert client.list_incidents(ctx, filters={"agent": "codex-repo"}, on_filters_unavailable=lambda: warnings.append(True)) == [] assert warnings == [True]