diff --git a/.github/scripts/test_public_boundary.py b/.github/scripts/test_public_boundary.py index 8d49575..03ffbe8 100644 --- a/.github/scripts/test_public_boundary.py +++ b/.github/scripts/test_public_boundary.py @@ -18,7 +18,7 @@ class PublicBoundaryTest(unittest.TestCase): def test_repository_satisfies_public_boundary(self) -> None: - self.assertEqual(boundary.verify(REPOSITORY_ROOT), "1.0.4") + self.assertEqual(boundary.verify(REPOSITORY_ROOT), "1.0.5") def test_forbidden_private_source_is_detected(self) -> None: with tempfile.TemporaryDirectory(prefix="public-boundary-") as temporary: diff --git a/Cargo.lock b/Cargo.lock index cf0587e..c1149b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1593,7 +1593,7 @@ checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "mcp-acceleration-products" -version = "1.0.4" +version = "1.0.5" dependencies = [ "async-trait", "chrono", @@ -1611,7 +1611,7 @@ dependencies = [ [[package]] name = "mcp-client" -version = "1.0.4" +version = "1.0.5" dependencies = [ "anyhow", "base64", @@ -1639,7 +1639,7 @@ dependencies = [ [[package]] name = "mcp-model-registry" -version = "1.0.4" +version = "1.0.5" dependencies = [ "mcp-types", "serde", @@ -1648,7 +1648,7 @@ dependencies = [ [[package]] name = "mcp-server" -version = "1.0.4" +version = "1.0.5" dependencies = [ "anyhow", "axum", @@ -1702,7 +1702,7 @@ dependencies = [ [[package]] name = "mcp-session" -version = "1.0.4" +version = "1.0.5" dependencies = [ "chrono", "dashmap", @@ -1721,7 +1721,7 @@ dependencies = [ [[package]] name = "mcp-tools" -version = "1.0.4" +version = "1.0.5" dependencies = [ "anyhow", "async-trait", @@ -1756,7 +1756,7 @@ dependencies = [ [[package]] name = "mcp-types" -version = "1.0.4" +version = "1.0.5" dependencies = [ "async-trait", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 31cdd7a..b5a9a87 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ members = [ ] [workspace.package] -version = "1.0.4" +version = "1.0.5" edition = "2021" rust-version = "1.95" license = "MIT" @@ -90,12 +90,12 @@ fs2 = "0.4" notify = "6" # Workspace crates -mcp-types = { version = "=1.0.4", path = "crates/mcp-types" } -mcp-client = { version = "=1.0.4", path = "crates/mcp-client" } -mcp-session = { version = "=1.0.4", path = "crates/mcp-session" } -mcp-tools = { version = "=1.0.4", path = "crates/mcp-tools" } -mcp-model-registry = { version = "=1.0.4", path = "crates/mcp-model-registry" } -mcp-acceleration-products = { version = "=1.0.4", path = "crates/mcp-acceleration-products" } +mcp-types = { version = "=1.0.5", path = "crates/mcp-types" } +mcp-client = { version = "=1.0.5", path = "crates/mcp-client" } +mcp-session = { version = "=1.0.5", path = "crates/mcp-session" } +mcp-tools = { version = "=1.0.5", path = "crates/mcp-tools" } +mcp-model-registry = { version = "=1.0.5", path = "crates/mcp-model-registry" } +mcp-acceleration-products = { version = "=1.0.5", path = "crates/mcp-acceleration-products" } # Testing mockall = "0.13" diff --git a/crates/mcp-tools/src/domains/display_title.rs b/crates/mcp-tools/src/domains/display_title.rs index 3a52aa9..6b5d868 100644 --- a/crates/mcp-tools/src/domains/display_title.rs +++ b/crates/mcp-tools/src/domains/display_title.rs @@ -214,6 +214,7 @@ pub fn normalize_recall_result_item(item: &mut Value) { ("event_id", "event_id"), ("transcript_id", "transcript_id"), ("doc_id", "doc_id"), + ("node_id", "node_id"), ("event_type", "event_type"), ("original_type", "kind"), ("node_type", "node_type"), @@ -232,8 +233,24 @@ pub fn normalize_recall_result_item(item: &mut Value) { } } - // MemorySearchResult id is the event/node uuid; surface as event_id when missing. - if obj.get("event_id").is_none() { + // UUIDs are table-local identities, not evidence that a source is an event. + // In particular, promoting a doc/node UUID to event_id changes follow-up + // reads and can collide with a real event carrying that same UUID. + let source_kind = obj + .get("source_kind") + .or_else(|| obj.get("result_type")) + .and_then(Value::as_str) + .unwrap_or("") + .to_ascii_lowercase(); + if obj.get("event_id").is_none() + && obj.get("doc_id").is_none() + && obj.get("node_id").is_none() + && obj.get("transcript_id").is_none() + && !matches!( + source_kind.as_str(), + "doc" | "document" | "node" | "knowledge_node" | "knowledgenode" | "transcript" + ) + { if let Some(id) = obj.get("id").cloned() { obj.insert("event_id".to_string(), id); } @@ -283,6 +300,17 @@ mod tests { assert_eq!(item["event_id"], "evt-uuid"); } + #[test] + fn normalization_preserves_document_and_node_identities() { + for (kind, field) in [("doc", "doc_id"), ("knowledge_node", "node_id")] { + let mut item = json!({"id":"same-uuid", "result_type":kind, + "metadata": {field:"same-uuid", "title":"stadium graphics"}}); + normalize_recall_result_item(&mut item); + assert_eq!(item[field], "same-uuid"); + assert!(item.get("event_id").is_none()); + } + } + #[test] fn search_keywords_from_content_preview_when_title_junk() { let item = json!({ diff --git a/crates/mcp-tools/src/domains/grounding.rs b/crates/mcp-tools/src/domains/grounding.rs index 5d391d1..ca443bf 100644 --- a/crates/mcp-tools/src/domains/grounding.rs +++ b/crates/mcp-tools/src/domains/grounding.rs @@ -120,6 +120,57 @@ pub fn replay_selection( ) } +/// Candidate serving must not leak rejected supplements through the structured +/// half of a response after its text has already used the qualified selector. +pub fn retain_selected_payload(recall: &mut Value, hits: &[GroundingHit]) { + remove_superseded_core_projections(recall); + let query = CandidateQueryEvidence::from_payload(recall); + let mut seen = std::collections::HashSet::new(); + for field in ["results", "supplemental_results"] { + if let Some(items) = recall.get_mut(field).and_then(Value::as_array_mut) { + items.retain(|item| { + let (id, id_field) = id_hint(item); + let project = retrieval_provenance(item) + .and_then(|p| p.get("source_scope")) + .and_then(|s| s.get("project_id")) + .and_then(Value::as_str); + id.is_some() + && candidate_evidence_admits(retrieval_provenance(item), query.as_ref()) + && hits.iter().any(|hit| { + hit.id_hint == id + && hit.id_field == id_field + && hit.source_project_id.as_deref() == project + && hit.retrieval_provenance.as_ref() == retrieval_provenance(item) + && hit.title == extract_display_title(item) + }) + && seen.insert((source_identity_kind(item), id, project.map(str::to_owned))) + }); + } + } +} + +fn remove_superseded_core_projections(recall: &mut Value) { + // A freshly authorized source (including a revoked/deleted source omitted + // by the authority) replaces its older core-cache projection BEFORE + // admission. Otherwise stale matching bytes could defeat a fresh miss. + let sources = recall + .get("supplemental_source_ids") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + if let Some(results) = recall.get_mut("results").and_then(Value::as_array_mut) { + results.retain(|item| { + !sources.iter().any(|source| { + source["kind"].as_str() == Some(source_identity_kind(item).as_str()) + && source["id"].as_str() + == metadata_str(item, "source_entity_id") + .or_else(|| metadata_str(item, "id")) + .as_deref() + }) + }); + } +} + fn recall_with_mode( mut recall: Value, session_id: Option<&str>, @@ -130,16 +181,32 @@ fn recall_with_mode( if !recall.get("results").is_some_and(Value::is_array) { return GroundingRecall::unavailable(); } + if mode != "shadow" + && recall.get("supplemental_status").and_then(Value::as_str) == Some("unavailable") + { + let mut unavailable = GroundingRecall::unavailable(); + unavailable.selection_mode = mode; + return unavailable; + } let hits = parse_recall_results(&recall); let query_evidence = CandidateQueryEvidence::from_payload(&recall); // Work on a private copy. Current-session evidence can break ties only // after the candidate proves query relevance. No active directives are // synthesized from recalled approvals or old permission requests. let mut shadow_payload = recall.clone(); + remove_superseded_core_projections(&mut shadow_payload); + // Only the dedicated primary-authority endpoint populates this field. + // Raw doc_matches/decision_matches remain legacy output, never evidence. + let supplemental = shadow_payload + .get("supplemental_results") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); if let Some(results) = shadow_payload .get_mut("results") .and_then(Value::as_array_mut) { + results.extend(supplemental); let mut seen = std::collections::HashSet::new(); results.retain(|item| { // An irrelevant projection must not consume the canonical identity @@ -155,22 +222,34 @@ fn recall_with_mode( query_evidence .as_ref() .and_then(|query| query.source_project(retrieval_provenance(item)?)), + source_identity_kind(item), id, )) }) .unwrap_or(true) }); - if let Some(session) = session_id.filter(|s| !s.is_empty()) { - results.sort_by(|a, b| { - item_score(b) - .partial_cmp(&item_score(a)) - .unwrap_or(Ordering::Equal) - .then_with(|| { + // Cross-source ordinal scores are not calibrated. Use the same final + // display evidence for all sources; stable sort retains discovery order + // on equal coverage, with current-session identity only a tie-breaker. + results.sort_by(|a, b| { + let coverage = |item: &Value| { + retrieval_provenance(item) + .and_then(|e| e.get("lexical_query_coverage")) + .and_then(Value::as_f64) + .unwrap_or(0.0) + }; + coverage(b) + .partial_cmp(&coverage(a)) + .unwrap_or(Ordering::Equal) + .then_with(|| { + if let Some(session) = session_id.filter(|s| !s.is_empty()) { (metadata_str(a, "session_id").as_deref() != Some(session)) .cmp(&(metadata_str(b, "session_id").as_deref() != Some(session))) - }) - }); - } + } else { + Ordering::Equal + } + }) + }); } let shadow = parse_recall_results_with_policy(&shadow_payload, true); let legacy_hit_count = hits.len(); @@ -180,6 +259,7 @@ fn recall_with_mode( candidate.id_hint.is_some() && hits.iter().any(|legacy| { legacy.id_hint == candidate.id_hint + && legacy.id_field == candidate.id_field && legacy.source_project_id == candidate.source_project_id }) }) @@ -528,6 +608,96 @@ fn classify_kind(item: &Value) -> String { .unwrap_or_else(|| "hit".to_string()) } +fn source_identity_kind(item: &Value) -> String { + if let Some(kind) = metadata_str(item, "source_kind") { + return kind; + } + if metadata_str(item, "transcript_id").is_some() { + return "transcript".into(); + } + if metadata_str(item, "doc_id").is_some() { + return "doc".into(); + } + if metadata_str(item, "node_id").is_some() { + return "node".into(); + } + match item + .get("result_type") + .and_then(Value::as_str) + .unwrap_or("") + .to_ascii_lowercase() + .as_str() + { + "knowledge_node" | "knowledgenode" | "node" => "node".into(), + "doc" | "document" => "doc".into(), + _ => "event".into(), + } +} + +#[test] +fn supplemental_evidence_enters_candidate_only_and_keeps_typed_ids() { + let mut payload = scoped_test_payload(serde_json::json!({"results":[ + {"id":"same","result_type":"event","score":0.99,"metadata":{"event_id":"same","title":"Routing","retrieval_provenance":{"query_term_matches":1}}}, + {"id":"same","result_type":"doc","score":0.0,"metadata":{"doc_id":"same","title":"Routing history","retrieval_provenance":{"query_term_matches":2}}}, + {"id":"same","result_type":"knowledge_node","score":0.0,"metadata":{"node_id":"same","summary":"Routing history","retrieval_provenance":{"query_term_matches":2}}} + ]})); + let supplements = payload["results"].as_array_mut().unwrap().split_off(1); + payload["supplemental_results"] = serde_json::json!(supplements); + let legacy = replay_selection(payload.clone(), None, false); + assert_eq!(legacy.hits.len(), 1); + let candidate = replay_selection(payload.clone(), None, true); + assert_eq!(candidate.hits.len(), 3); + assert_eq!(candidate.hits[0].id_field.as_deref(), Some("doc_id")); + assert_eq!(candidate.hits[1].id_field.as_deref(), Some("node_id")); + let text = format_grounding_block(&candidate.hits, true); + assert!(text.contains("get_doc")); + assert!(text.contains("get_node")); + payload["supplemental_results"][0]["metadata"]["retrieval_provenance"] = Value::Null; + let candidate = replay_selection(payload.clone(), None, true); + assert_eq!(candidate.hits.len(), 2); + let mut display = payload.clone(); + retain_selected_payload(&mut display, &candidate.hits); + assert_eq!(display["supplemental_results"].as_array().unwrap().len(), 1); + assert_eq!( + display["supplemental_results"][0]["metadata"]["node_id"], + "same" + ); + payload["supplemental_status"] = serde_json::json!("unavailable"); + assert_eq!(replay_selection(payload, None, true).status, "unavailable"); +} + +#[test] +fn fresh_authority_replaces_stale_core_even_when_it_revokes_or_removes_relevance() { + let mut payload = scoped_test_payload(serde_json::json!({"results":[{ + "id":"same","result_type":"event","score":0.99, + "metadata":{"event_id":"same","title":"Stale matching bytes","retrieval_provenance":{"query_term_matches":2}} + }]})); + payload["supplemental_source_ids"] = serde_json::json!([{"kind":"event","id":"same"}]); + payload["supplemental_results"] = serde_json::json!([]); + assert_eq!(replay_selection(payload.clone(), None, false).hits.len(), 1); + assert!(replay_selection(payload.clone(), None, true) + .hits + .is_empty()); + let mut fresh = payload["results"][0].clone(); + fresh["metadata"]["title"] = serde_json::json!("Fresh primary bytes"); + payload["supplemental_results"] = serde_json::json!([fresh]); + let selected = replay_selection(payload.clone(), None, true); + assert_eq!(selected.hits.len(), 1); + assert_eq!(selected.hits[0].title, "Fresh primary bytes"); + let mut displayed = payload.clone(); + retain_selected_payload(&mut displayed, &selected.hits); + assert!(displayed["results"].as_array().unwrap().is_empty()); + assert_eq!( + displayed["supplemental_results"].as_array().unwrap().len(), + 1 + ); + payload["supplemental_results"][0]["metadata"]["retrieval_provenance"]["query_term_matches"] = + serde_json::json!(0); + payload["supplemental_results"][0]["metadata"]["retrieval_provenance"] + ["lexical_query_coverage"] = serde_json::json!(0.0); + assert!(replay_selection(payload, None, true).hits.is_empty()); +} + fn id_hint(item: &Value) -> (Option, Option) { for key in [ "transcript_id", @@ -535,6 +705,7 @@ fn id_hint(item: &Value) -> (Option, Option) { "event_id", "memory_event_id", "doc_id", + "node_id", "feed_id", "id", ] { @@ -824,6 +995,9 @@ fn action_hint(hit: &GroundingHit, search_keywords: &str) -> String { if is_doc_id_field(id_field) && !id.is_empty() { return format!("memory(action=\"get_doc\", doc_id=\"{id}\")"); } + if id_field == "node_id" && !id.is_empty() { + return format!("memory(action=\"get_node\", node_id=\"{id}\")"); + } if (k.contains("transcript") || k.contains("conversation") || k == "session") && !is_event_id_field(id_field) diff --git a/crates/mcp-tools/src/domains/grounding_rollout.rs b/crates/mcp-tools/src/domains/grounding_rollout.rs index dc19aa9..9e8a89a 100644 --- a/crates/mcp-tools/src/domains/grounding_rollout.rs +++ b/crates/mcp-tools/src/domains/grounding_rollout.rs @@ -2,7 +2,10 @@ //! Default/invalid configuration and the independent kill switch serve legacy. use serde::Deserialize; -pub const POLICY_REVISION: &str = "grounding-evidence-v2"; +pub const POLICY_REVISION: &str = "grounding-evidence-v3"; +// Freeze the existing population even when a new selector requires fresh +// qualification. Changing policy must not silently move sticky cohorts. +const COHORT_NAMESPACE: &str = "grounding-evidence-v2"; // Independent of selector policy: changing scoring must not reshuffle cohorts. pub const EVALUATION_REVISION: &str = "grounding-quality-v2"; @@ -172,7 +175,7 @@ impl Rollout { fn cohort_bucket(salt: &str, subject: &str, workspace: &str, project: Option<&str>) -> u64 { let mut hash = 0xcbf29ce484222325u64; for field in [ - POLICY_REVISION, + COHORT_NAMESPACE, salt, subject, workspace, diff --git a/crates/mcp-tools/src/domains/recall_supplements.rs b/crates/mcp-tools/src/domains/recall_supplements.rs new file mode 100644 index 0000000..ca6abae --- /dev/null +++ b/crates/mcp-tools/src/domains/recall_supplements.rs @@ -0,0 +1,361 @@ +//! Admit supplemental sources only through fresh primary authorization. +use mcp_client::ContextStreamClient; +use mcp_types::{Error, Result}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::{collections::HashSet, time::Duration}; +use uuid::Uuid; + +fn invalid() -> Error { + Error::Validation("Supplemental recall evidence is unavailable or malformed".into()) +} + +fn source_ids(decisions: &[Value], docs: &[Value]) -> Result> { + let mut sources = Vec::new(); + let mut seen = HashSet::new(); + for (kind, item) in docs.iter().map(|item| (Some("doc"), item)).chain( + decisions + .iter() + .map(|item| (item.get("source").and_then(Value::as_str), item)), + ) { + let kind = kind.ok_or_else(invalid)?.to_ascii_lowercase(); + if !matches!(kind.as_str(), "doc" | "event" | "node") { + return Err(invalid()); + } + let id = item + .get("id") + .and_then(Value::as_str) + .and_then(|id| Uuid::parse_str(id).ok()) + .ok_or_else(invalid)?; + if seen.insert((kind.clone(), id)) { + sources.push(json!({"kind":kind,"id":id})); + } + } + if sources.len() > 20 { + return Err(invalid()); + } + Ok(sources) +} + +fn checked_results( + value: Value, + workspace: Uuid, + project: Option, + query: &str, + sources: &[Value], +) -> Result> { + if value["evidence_contract"] != "supplemental_primary_v1" + || value["query"] != query + || value["workspace_id"] != json!(workspace) + || value.get("project_id") != Some(&json!(project)) + || value.get("degraded") != Some(&Value::Bool(false)) + || !value + .get("errors") + .and_then(Value::as_array) + .is_some_and(Vec::is_empty) + { + return Err(invalid()); + } + let results = value + .get("results") + .and_then(Value::as_array) + .ok_or_else(invalid)?; + let mut seen = HashSet::new(); + for result in results { + let id = result + .get("id") + .and_then(Value::as_str) + .ok_or_else(invalid)?; + let kind = result + .get("source_kind") + .and_then(Value::as_str) + .ok_or_else(invalid)?; + if !seen.insert((kind, id)) || !sources.iter().any(|s| s["kind"] == kind && s["id"] == id) { + return Err(invalid()); + } + let metadata = &result["metadata"]; + if metadata["source_kind"] != kind || metadata["source_entity_id"] != id { + return Err(invalid()); + } + // The selector separately verifies query digest, exact source scope, + // coverage and evidence version. Never promote a raw list response. + if metadata["retrieval_provenance"]["evidence_source"] != "primary_authorized_display" { + return Err(invalid()); + } + let provenance = &metadata["retrieval_provenance"]; + let source_scope = &provenance["source_scope"]; + let actual_project = source_scope + .get("project_id") + .and_then(|v| serde_json::from_value::>(v.clone()).ok()) + .ok_or_else(invalid)?; + let identity_field = match kind { + "doc" => "doc_id", + "event" => "event_id", + "node" => "node_id", + _ => return Err(invalid()), + }; + if metadata[identity_field] != id + || metadata["source_scope"] != *source_scope + || source_scope["workspace_id"] != json!(workspace) + || project + .zip(actual_project) + .is_some_and(|(requested, actual)| requested != actual) + || provenance["version"] != 2 + || provenance["query_sha256"] != format!("{:x}", Sha256::digest(query.as_bytes())) + || provenance["score_kind"] != "uncalibrated" + || provenance["calibration"] != "uncalibrated" + { + return Err(invalid()); + } + } + Ok(results.clone()) +} + +pub(super) fn mark_unavailable(recall: &mut Value) { + if let Some(object) = recall.as_object_mut() { + object.insert("supplemental_results".into(), json!([])); + object.insert("supplemental_source_ids".into(), json!([])); + object.insert("supplemental_status".into(), json!("unavailable")); + object.insert("degraded".into(), json!(true)); + let errors = object.entry("errors").or_insert_with(|| json!([])); + if let Some(errors) = errors.as_array_mut() { + errors.push(json!("supplemental_recall_unavailable")); + } + } +} + +/// Legacy formatting may keep its shape, but not pre-authorization bytes. +/// Do not copy current_truth or conflict claims from a stale discovery row. +pub(super) fn display_views(recall: &Value) -> (Vec, Vec) { + let mut decisions = Vec::new(); + let mut docs = Vec::new(); + if let Some(results) = recall.get("supplemental_results").and_then(Value::as_array) { + for result in results { + let Some(mut display) = result.get("metadata").and_then(Value::as_object).cloned() + else { + continue; + }; + display.insert("id".into(), result["id"].clone()); + display.insert( + "project_id".into(), + display + .get("source_project_id") + .cloned() + .unwrap_or(Value::Null), + ); + if result["source_kind"] == "doc" { + docs.push(Value::Object(display)); + } else { + display.insert("source".into(), result["source_kind"].clone()); + if !display.contains_key("summary") { + display.insert( + "summary".into(), + display.get("title").cloned().unwrap_or(Value::Null), + ); + } + if !display.contains_key("details") { + display.insert( + "details".into(), + display + .get("content_preview") + .cloned() + .unwrap_or(Value::Null), + ); + } + decisions.push(Value::Object(display)); + } + } + } + (decisions, docs) +} + +pub(super) async fn attach( + client: &ContextStreamClient, + workspace: Option, + project: Option, + query: &str, + decisions: &[Value], + docs: &[Value], + recall: &mut Value, +) { + let attempt = + async { + let workspace = workspace.ok_or_else(invalid)?; + let sources = source_ids(decisions, docs)?; + if sources.is_empty() { + return Ok((Vec::new(), sources)); + } + // Never cache this response: docs do not carry memory cache revisions. + let value: Value = client.post("/session/recall/evidence", json!({ + "query":query,"workspace_id":workspace,"project_id":project,"sources":sources + })).await?; + checked_results(value, workspace, project, query, &sources) + .map(|results| (results, sources)) + }; + match tokio::time::timeout(Duration::from_millis(1800), attempt).await { + Ok(Ok((results, sources))) => { + if let Some(object) = recall.as_object_mut() { + object.insert("supplemental_results".into(), json!(results)); + object.insert("supplemental_source_ids".into(), json!(sources)); + object.insert("supplemental_status".into(), json!("available")); + } + } + _ => mark_unavailable(recall), + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn inputs_are_only_typed_ids_and_collisions_are_not_deduplicated() { + let id = Uuid::new_v4(); + let decisions = vec![ + json!({"id":id,"source":"event","summary":"untrusted"}), + json!({"id":id,"source":"node"}), + ]; + let docs = vec![json!({"id":id,"content":"untrusted"}), json!({"id":id})]; + let sources = source_ids(&decisions, &docs).unwrap(); + assert_eq!(sources.len(), 3); + assert!(sources.iter().all(|s| s.as_object().unwrap().len() == 2)); + assert!(source_ids(&[json!({"id":id})], &[]).is_err()); + } + #[test] + fn old_or_malformed_server_responses_cannot_become_evidence() { + assert!( + checked_results(json!({"results":[]}), Uuid::nil(), None, "graphics", &[]).is_err() + ); + let mut recall = json!({"results":[],"degraded":false,"errors":[]}); + mark_unavailable(&mut recall); + assert_eq!(recall["degraded"], true); + assert_eq!(recall["supplemental_status"], "unavailable"); + } + + fn response(workspace: Uuid, id: Uuid, content: &str) -> Value { + let scope = json!({"workspace_id":workspace,"project_id":null}); + json!({"evidence_contract":"supplemental_primary_v1", "workspace_id":workspace, + "project_id":null,"query":"graphics","degraded":false,"errors":[], "results":[{ + "id":id,"source_kind":"doc","result_type":"doc","score":0.0, + "metadata":{"doc_id":id,"source_kind":"doc","source_entity_id":id, + "title":"graphics","content_preview":content,"source_scope":scope,"source_project_id":null, + "retrieval_provenance":{"version":2,"evidence_source":"primary_authorized_display", + "score_kind":"uncalibrated","calibration":"uncalibrated","source_scope":scope, + "query_sha256":format!("{:x}",Sha256::digest(b"graphics")), + "query_term_matches":1,"query_term_count":1,"lexical_query_coverage":1.0}}}]}) + } + + #[test] + fn responses_require_exact_query_identity_and_actual_source_scope() { + let ws = Uuid::new_v4(); + let id = Uuid::new_v4(); + let sources = vec![json!({"kind":"doc","id":id})]; + let valid = response(ws, id, "primary"); + assert_eq!( + checked_results(valid.clone(), ws, None, "graphics", &sources) + .unwrap() + .len(), + 1 + ); + for (path, replacement) in [ + ("/query", json!("different")), + ("/results/0/metadata/doc_id", json!(Uuid::new_v4())), + ( + "/results/0/metadata/retrieval_provenance/query_sha256", + json!("0".repeat(64)), + ), + ( + "/results/0/metadata/retrieval_provenance/source_scope", + json!({"workspace_id":ws}), + ), + ( + "/results/0/metadata/source_scope/workspace_id", + json!(Uuid::new_v4()), + ), + ] { + let mut invalid = valid.clone(); + *invalid.pointer_mut(path).unwrap() = replacement; + assert!( + checked_results(invalid, ws, None, "graphics", &sources).is_err(), + "accepted {path}" + ); + } + } + + #[tokio::test] + async fn repeated_calls_reauthorize_and_never_replay_revoked_document_bytes() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let ws = Uuid::new_v4(); + let id = Uuid::new_v4(); + let server = tokio::spawn(async move { + for turn in 0..3 { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut bytes = Vec::new(); + loop { + let mut chunk = [0u8; 4096]; + let count = socket.read(&mut chunk).await.unwrap(); + assert!(count > 0); + bytes.extend_from_slice(&chunk[..count]); + assert!(bytes.len() < 32768); + if let Some(end) = bytes.windows(4).position(|w| w == b"\r\n\r\n") { + let headers = String::from_utf8_lossy(&bytes[..end]); + assert!(headers.starts_with("POST /api/v1/session/recall/evidence ")); + let length: usize = headers + .lines() + .filter_map(|line| line.split_once(':')) + .find(|(key, _)| key.eq_ignore_ascii_case("content-length")) + .unwrap() + .1 + .trim() + .parse() + .unwrap(); + if bytes.len() >= end + 4 + length { + let request: Value = + serde_json::from_slice(&bytes[end + 4..end + 4 + length]).unwrap(); + assert_eq!(request["sources"], json!([{"kind":"doc","id":id}])); + assert_eq!(request.as_object().unwrap().len(), 4); + break; + } + } + } + let mut payload = response(ws, id, &format!("primary version {turn}")); + if turn == 2 { + payload["results"] = json!([]); + } + let payload = payload.to_string(); + socket.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",payload.len(),payload).as_bytes()).await.unwrap(); + } + }); + let mut config = crate::testing::TestFixtures::test_config(); + config.api_url = format!("http://{address}"); + let client = ContextStreamClient::new(config); + for turn in 0..3 { + let mut recall = json!({"results":[]}); + attach( + &client, + Some(ws), + None, + "graphics", + &[], + &[json!({"id":id,"content":"stale discovery bytes"})], + &mut recall, + ) + .await; + assert_eq!(recall["supplemental_status"], "available"); + let (_, docs) = display_views(&recall); + if turn == 2 { + assert!(docs.is_empty()); + } else { + assert_eq!( + docs[0]["content_preview"], + format!("primary version {turn}") + ); + } + } + tokio::time::timeout(Duration::from_secs(3), server) + .await + .unwrap() + .unwrap(); + } +} diff --git a/crates/mcp-tools/src/domains/session.rs b/crates/mcp-tools/src/domains/session.rs index f7fb015..83eae8f 100644 --- a/crates/mcp-tools/src/domains/session.rs +++ b/crates/mcp-tools/src/domains/session.rs @@ -1,5 +1,8 @@ //! Session domain tools: init, context, capture, recall, compress. +#[path = "recall_supplements.rs"] +mod recall_supplements; + use async_trait::async_trait; use mcp_client::{ format_linked_summary, get_task_auth_override, normalize_linked_items_with_allowed_kinds, @@ -6594,48 +6597,82 @@ async fn proactive_grounding_recall( user_scope: user_scope.map(|s| s.to_string()), }; - if let Some(bundle) = crate::domains::atlas_warm_cache::try_lookup( - atlas_layer, - mcp_types::atlas_layer::AtlasWarmCacheKind::Recall, - scope, - 1000, // primary baseline ms — recall p95 ≈ 1s - ) - .await - { - return recall_with_rollout( - bundle.payload.clone(), - session_id, - user_scope, - Some(&ws.to_string()), - project_id.map(|p| p.to_string()).as_deref(), - ); - } + let primary = async { + if let Some(bundle) = crate::domains::atlas_warm_cache::try_lookup( + atlas_layer, + mcp_types::atlas_layer::AtlasWarmCacheKind::Recall, + scope, + 1000, // primary baseline ms — recall p95 ≈ 1s + ) + .await + { + return Some(bundle.payload); + } - // Cache miss: run the primary recall, then write back so the next - // `context()` turn (and any user-facing `session(recall)` call - // for the same scope) hits. - match tokio::time::timeout(grounding_timeout(), client.session_recall(params)).await { - Ok(Ok(value)) => { - let scope_for_put = mcp_types::atlas_layer::AtlasFederationScope { - workspace_id: ws, + // Cache miss: run the primary recall, then write back so the next + // `context()` turn (and any user-facing `session(recall)` call + // for the same scope) hits. + match tokio::time::timeout(grounding_timeout(), client.session_recall(params)).await { + Ok(Ok(value)) => { + let scope_for_put = mcp_types::atlas_layer::AtlasFederationScope { + workspace_id: ws, + project_id, + scope_hash, + user_scope: user_scope.map(|s| s.to_string()), + }; + crate::domains::atlas_warm_cache::put_in_background( + atlas_layer.clone(), + mcp_types::atlas_layer::AtlasWarmCacheKind::Recall, + scope_for_put, + value.clone(), + ); + Some(value) + } + _ => None, + } + }; + // Atlas may reuse core recall, but never supplemental display bytes. Both + // cold and warm paths do fresh typed discovery and primary authorization. + let collect = async { + let (primary, (decisions, docs)) = tokio::join!( + primary, + search_recall_augmentations( + client, + workspace_id, project_id, - scope_hash, - user_scope: user_scope.map(|s| s.to_string()), - }; - crate::domains::atlas_warm_cache::put_in_background( - atlas_layer.clone(), - mcp_types::atlas_layer::AtlasWarmCacheKind::Recall, - scope_for_put, - value.clone(), - ); - recall_with_rollout( - value, - session_id, - user_scope, - Some(&ws.to_string()), - project_id.map(|p| p.to_string()).as_deref(), + user_message, + 5, + 5, + true, + true ) + ); + let mut value = primary?; + match (decisions, docs) { + (Ok(decisions), Ok(docs)) => { + recall_supplements::attach( + client, + workspace_id, + project_id, + user_message, + &decisions, + &docs, + &mut value, + ) + .await + } + _ => recall_supplements::mark_unavailable(&mut value), } + Some(value) + }; + match tokio::time::timeout(grounding_timeout(), collect).await { + Ok(Some(value)) => recall_with_rollout( + value, + session_id, + user_scope, + Some(&ws.to_string()), + project_id.map(|p| p.to_string()).as_deref(), + ), _ => GroundingRecall::unavailable(), } } @@ -9826,10 +9863,13 @@ where account_block ); - let recall = recall.unwrap_or_else(|err| { + let mut recall = recall.unwrap_or_else(|err| { tracing::debug!("session ground: recall failed: {}", err); serde_json::json!({}) }); + if decisions.is_err() || docs.is_err() { + recall_supplements::mark_unavailable(&mut recall); + } let decisions = decisions.unwrap_or_else(|err| { tracing::debug!("session ground: decision augmentation failed: {}", err); Vec::new() @@ -10827,15 +10867,20 @@ impl ToolHandler for SessionRecallTool { .flatten(); if let Some(cache_key) = cache_key.as_deref() { if let Some((cached_text, cached_structured)) = recall_cache().get(cache_key) { - tracing::debug!("recall cache hit: key={}", cache_key); - let marked = format!( + if cached_structured.get("supplemental_results").is_none() + && cached_structured.get("doc_matches").is_none() + && cached_structured.get("decision_matches").is_none() + { + tracing::debug!("recall cache hit: key={}", cache_key); + let marked = format!( "[RECALL_CACHED] Same recall query as the previous identical call (<{}s ago); \ returning cached result. Change the query/toggles to refresh.\n\n{}", RECALL_CACHE_TTL.as_secs(), cached_text ); - consume_grounding_session(&self.session).await; - return Ok(ToolResult::with_structured(marked, cached_structured)); + consume_grounding_session(&self.session).await; + return Ok(ToolResult::with_structured(marked, cached_structured)); + } } } @@ -10933,6 +10978,7 @@ impl ToolHandler for SessionRecallTool { join_recall_with_augmentations(primary_recall, decision_search, doc_search).await?; let mut result = result; + let supplemental_discovery_failed = doc_matches.is_err(); crate::domains::display_title::normalize_recall_payload(&mut result); let count = result @@ -10955,6 +11001,20 @@ impl ToolHandler for SessionRecallTool { if count > 0 { doc_matches.truncate(3); } + recall_supplements::attach( + &self.client, + scope.workspace_id, + scope.project_id, + &input.query, + &decision_matches, + &doc_matches, + &mut result, + ) + .await; + if supplemental_discovery_failed { + recall_supplements::mark_unavailable(&mut result); + } + let (decision_matches, doc_matches) = recall_supplements::display_views(&result); let project_matches = if !recall_checkout_scope_unroutable && input.include_related.unwrap_or(true) && (count == 0 || (decision_matches.is_empty() && doc_matches.is_empty())) @@ -10981,6 +11041,14 @@ impl ToolHandler for SessionRecallTool { let has_decision_matches = !decision_matches.is_empty(); let has_doc_matches = !doc_matches.is_empty(); let has_project_matches = !project_matches.is_empty(); + let selection_session = self.session.state().await.session_id; + let selection = crate::domains::grounding::recall_with_rollout( + result.clone(), + selection_session.as_deref(), + caller_cache_identity.as_deref(), + scope.workspace_id.map(|id| id.to_string()).as_deref(), + scope.project_id.map(|id| id.to_string()).as_deref(), + ); let memory_items = result .get("results") .and_then(|v| v.as_array()) @@ -10993,6 +11061,12 @@ impl ToolHandler for SessionRecallTool { &doc_matches, &project_matches, ); + if selection.selection_mode != "shadow" { + text = crate::domains::grounding::format_grounding_block(&selection.hits, false); + } + if result.get("supplemental_status").and_then(Value::as_str) == Some("unavailable") { + text = format!("[GROUNDING_UNAVAILABLE] Supplemental source validation was unavailable; do not interpret this as no prior documents or decisions.\n\n{text}"); + } if recall_checkout_scope_unroutable { text = format!( "[CHECKOUT_SCOPE] Recall used durable project memory, but skipped source-code augmentation because the MCP could not derive an exact active-checkout locator.\n\n{text}" @@ -11000,6 +11074,13 @@ impl ToolHandler for SessionRecallTool { } let mut structured = result; + if selection.selection_mode != "shadow" { + crate::domains::grounding::retain_selected_payload(&mut structured, &selection.hits); + } + if let Some(object) = structured.as_object_mut() { + object.insert("grounding_retrieval".into(), selection.telemetry()); + object.insert("grounding_hits".into(), serde_json::json!(selection.hits)); + } if recall_checkout_scope_unroutable { if let Some(object) = structured.as_object_mut() { object.insert("checkout_scope_unconfirmed".to_string(), Value::Bool(true)); @@ -11009,7 +11090,9 @@ impl ToolHandler for SessionRecallTool { ); } } - if has_decision_matches || has_doc_matches || has_project_matches { + if selection.selection_mode == "shadow" + && (has_decision_matches || has_doc_matches || has_project_matches) + { if let Some(obj) = structured.as_object_mut() { if has_decision_matches { obj.insert( @@ -11050,7 +11133,9 @@ impl ToolHandler for SessionRecallTool { // Cache the rendered result for repeat recall calls inside the // warm window. Scope note is NOT included in the cached text so // the prefix stays consistent across calls with/without the note. - if let Some(cache_key) = cache_key { + if let Some(cache_key) = + cache_key.filter(|_| structured.get("supplemental_results").is_none()) + { put_recall_cache( caller_cache_identity.as_deref(), cache_key, @@ -15297,22 +15382,19 @@ async fn consume_grounding_session(session: &Arc) { /// Upper bound on the Context Feeds grounding read inside `session(ground)`. const FEED_GROUNDING_TIMEOUT_MS: u64 = 2_000; -fn composite_ground_cache_eligible(input: &SessionInput, has_checkout: bool) -> bool { - // This cache contains a formatted bundle, including checkout state and - // session-selected evidence. Only the default, workspace-only read is - // shareable by its existing key. Raw recall remains independently cached. - !has_checkout - && !crate::domains::grounding::rollout::configured() - && input.session_id.is_none() - && input.include_decisions.unwrap_or(true) - && input.include_related.unwrap_or(true) +fn composite_ground_cache_eligible(_input: &SessionInput, _has_checkout: bool) -> bool { + // Persisted Ground entries can include document/decision display bytes, + // but their cache contract cannot re-authorize those sources. Even an + // explicit omission shares a key with older default bundles. Keep the + // individually guarded core-recall cache; do not reuse formatted Ground. + false } #[test] fn composite_ground_cache_never_reuses_checkout_or_session_state() { let input: SessionInput = serde_json::from_value(serde_json::json!({"action":"ground"})).unwrap(); - assert!(composite_ground_cache_eligible(&input, false)); + assert!(!composite_ground_cache_eligible(&input, false)); assert!(!composite_ground_cache_eligible(&input, true)); for extra in [ serde_json::json!({"session_id":"session-a"}), @@ -15441,7 +15523,7 @@ async fn execute_session_ground( ) .await; let GroundingRemoteReads { - recall: recall_val, + recall: mut recall_val, decisions, docs, lessons, @@ -15450,6 +15532,25 @@ async fn execute_session_ground( account_block, } = remote_reads; + let supplemental_discovery_failed = recall_val + .get("supplemental_status") + .and_then(Value::as_str) + == Some("unavailable"); + recall_supplements::attach( + client, + scope.workspace_id, + scope.project_id, + user_message, + &decisions, + &docs, + &mut recall_val, + ) + .await; + if supplemental_discovery_failed { + recall_supplements::mark_unavailable(&mut recall_val); + } + let (decisions, docs) = recall_supplements::display_views(&recall_val); + let session_state = session.state().await; let fp = session_state.folder_path.clone(); let git_note = if let Some(ref p) = fp { @@ -15540,11 +15641,11 @@ async fn execute_session_ground( text.push_str(&crate::domains::grounding::format_grounding_block( hits, false, )); - if !decisions.is_empty() { + if grounding_recall.selection_mode == "shadow" && !decisions.is_empty() { text.push('\n'); text.push_str(&format_recall_decision_matches(&decisions)); } - if !docs.is_empty() { + if grounding_recall.selection_mode == "shadow" && !docs.is_empty() { text.push('\n'); text.push_str(&format_recall_doc_matches(&docs)); } @@ -15570,7 +15671,10 @@ async fn execute_session_ground( text.push_str(&account_block); } - let structured = serde_json::json!({ + if grounding_recall.selection_mode != "shadow" { + crate::domains::grounding::retain_selected_payload(&mut recall_val, hits); + } + let mut structured = serde_json::json!({ "recall": recall_val, "grounding_retrieval": grounding_recall.telemetry(), "decision_matches": decisions, @@ -15581,6 +15685,12 @@ async fn execute_session_ground( "grounding_hits": serde_json::to_value(hits).unwrap_or_else(|_| serde_json::json!([])), "feed_items": feed_items, }); + if grounding_recall.selection_mode != "shadow" { + if let Some(object) = structured.as_object_mut() { + object.remove("doc_matches"); + object.remove("decision_matches"); + } + } if let Some(ref p) = fp { grounding_state::clear_grounding_consumed(p); diff --git a/crates/mcp-tools/src/domains/session_tests.rs b/crates/mcp-tools/src/domains/session_tests.rs index f32b4a8..9330188 100644 --- a/crates/mcp-tools/src/domains/session_tests.rs +++ b/crates/mcp-tools/src/domains/session_tests.rs @@ -2617,7 +2617,9 @@ mod recall_augmentation_tests { ) .await; - assert_eq!(result.recall, json!({})); + assert_eq!(result.recall["supplemental_status"], "unavailable"); + assert_eq!(result.recall["degraded"], true); + assert!(result.recall.get("results").is_none()); assert!(result.decisions.is_empty()); assert!(result.docs.is_empty()); assert_eq!(result.lessons, json!({})); diff --git a/package.json b/package.json index 932ff42..864367a 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@contextstream/mcp-server", "mcpName": "io.github.contextstream/mcp-server", - "version": "1.0.4", + "version": "1.0.5", "description": "Verified npm launcher for the open-source ContextStream Rust MCP server", "type": "module", "license": "MIT", diff --git a/server.json b/server.json index 88b9aff..73c21e2 100644 --- a/server.json +++ b/server.json @@ -3,7 +3,7 @@ "name": "io.github.contextstream/mcp-server", "title": "ContextStream MCP Server", "description": "Project memory, semantic code search, and grounded agent context.", - "version": "1.0.4", + "version": "1.0.5", "repository": { "url": "https://github.com/contextstream/mcp-server", "source": "github" @@ -20,7 +20,7 @@ "registryType": "npm", "registryBaseUrl": "https://registry.npmjs.org", "identifier": "@contextstream/mcp-server", - "version": "1.0.4", + "version": "1.0.5", "transport": { "type": "stdio" }, diff --git a/testing/grounding/qualify.py b/testing/grounding/qualify.py index 818479a..aca9f60 100644 --- a/testing/grounding/qualify.py +++ b/testing/grounding/qualify.py @@ -14,7 +14,7 @@ import subprocess import time -POLICY = "grounding-evidence-v2" +POLICY = "grounding-evidence-v3" # Scoring evolves independently of selector policy and sticky cohort assignment. EVALUATION = "grounding-quality-v2" CATEGORIES = {"continuation", "paraphrase", "history", "supersession", "scope", "no_answer"}