From 87818dac1262bcf0fe26e32d510423f279dd81c0 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 03:46:37 +0000 Subject: [PATCH 01/84] test(ci): close three master-tip readiness races HTTP reset checks ride out a still-warming project open instead of treating the first after_delay answer as the terminal. Ignored-dependency publication faults a pointer a background pass cannot rename over. Killing a test daemon now signals its process group, and the next spawn waits out a socket that is still closing. Co-authored-by: Zack Jackson --- crates/tracedecay/tests/common/mod.rs | 18 +++ .../flight_tests.rs | 33 ++++- .../transport_boundaries.rs | 133 +++++++++++++----- 3 files changed, 143 insertions(+), 41 deletions(-) diff --git a/crates/tracedecay/tests/common/mod.rs b/crates/tracedecay/tests/common/mod.rs index 28bbc596c4..201a7a2b38 100644 --- a/crates/tracedecay/tests/common/mod.rs +++ b/crates/tracedecay/tests/common/mod.rs @@ -783,11 +783,29 @@ impl Drop for TestChildProcess { } /// PID-directed stop: survives `process_group(0)` / `setsid` detachment. +/// +/// The child is the leader of its own group. Killing only that pid leaves +/// helper children that still hold the listen socket, so the next spawn +/// observes a connectable daemon after this process has already been reaped. fn terminate_and_reap(child: &mut Child) -> std::io::Result { if let Ok(Some(status)) = child.try_wait() { return Ok(status); } + #[cfg(unix)] + { + let pid = child.id(); + if pid != 0 { + // SAFETY: `pid` is this live child. Negating it targets the + // process group `process_group(0)` created with that pid as + // leader. ESRCH is ignored: setpgid may not have run yet, and the + // pid kill below still stops the leader. + unsafe { + libc::kill(-(pid as i32), libc::SIGKILL); + } + } + } + if let Err(kill_err) = child.kill() { if let Some(status) = child.try_wait()? { return Ok(status); diff --git a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs index bc1388df7f..77309c5c22 100644 --- a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs +++ b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs @@ -2,6 +2,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Condvar, Mutex}; use tracedecay_code_index::production::CodeIndexProductionErrorV1; +use tracedecay_code_index_retention::code_index_generations::try_acquire_code_generation_store_lock; use super::*; @@ -157,6 +158,33 @@ fn assert_publication_error(error: CodeIndexSchedulerErrorV1) { ); } +/// Hold the only background permit once no pass is in flight. +/// +/// Text seating keeps `reconcile_in_progress` after it drops the scheduler +/// mutex, and that pass can still rename a valid active pointer. A truncated +/// pointer written in that window is not a closed fault. Occupying the permit +/// while the owner has not entered its pass stops that rewrite. +async fn hold_idle_background_admission( + registry: &CodeIndexSchedulerRegistryV1, +) -> tokio::sync::OwnedSemaphorePermit { + let admission = registry.background_reconcile_admission(); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + if registry.memory_stats().await.reconciling_worktrees == 0 + && let Ok(permit) = admission.clone().try_acquire_owned() + { + if registry.memory_stats().await.reconciling_worktrees == 0 { + return permit; + } + } + assert!( + std::time::Instant::now() <= deadline, + "background reconcile did not go idle before publication fault injection" + ); + tokio::time::sleep(Duration::from_millis(2)).await; + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn aborted_flight_owner_wakes_follower_and_allows_a_fresh_owner() { let fixture = fixture(); @@ -242,6 +270,8 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { let registry = Arc::new(mount(fixture.path(), &store, 1).await); let baseline = latest(®istry, fixture.path()).await; let request = request_for(&baseline, "pkg"); + let idle_admission = hold_idle_background_admission(®istry).await; + registry.clear_pending_wake_for_scope(&request.scope).await; let hold = SchedulerHold::acquire(®istry, fixture.path()).await; let (owner_control, owner_entered) = BlockingNthControl::new(4); @@ -297,8 +327,6 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { // and any writer that starts after it is released reads the corruption // under the lock and refuses instead of overwriting it. let pointer_bytes = { - use tracedecay_code_index_retention::code_index_generations::try_acquire_code_generation_store_lock; - let store_lock = tokio::time::timeout(Duration::from_secs(5), async { loop { if let Some(lock) = try_acquire_code_generation_store_lock(&scoped_store) @@ -316,6 +344,7 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { drop(store_lock); pointer_bytes }; + drop(idle_admission); owner_control.release(); hold.release(); diff --git a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/transport_boundaries.rs b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/transport_boundaries.rs index daca6a9303..dabdfd4a42 100644 --- a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/transport_boundaries.rs +++ b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/transport_boundaries.rs @@ -377,6 +377,62 @@ fn problem_envelope(payload: &Value, context: &str) -> Value { panic!("{context}: no typed problem envelope in the payload: {payload}") } +/// True when the daemon has not published the project open yet. +/// +/// The open wait on a connection is 500 ms. Past that, the surface answers +/// `unavailable` / `after_delay` and leaves the open running. The CLI rides +/// that refusal out; a raw HTTP or SDK call does not. A restart's first packet +/// can therefore be warming even when the store's recorded terminal is +/// `reset_required`. +fn retryable_pre_admission_unavailable(payload: &Value) -> bool { + let problem = &payload["problem"]; + problem["kind"] == "unavailable" + && problem["retry"] == "after_delay" + && problem["terminality"] == "pre_admission" +} + +/// Polls until the open publishes a non-retryable problem, then returns it. +/// +/// Bounded by the same 15 s a CLI tool call gives a cold open. A refusal that +/// stays retryable past that bound is a real failure, not a slow open. +fn await_settled_problem(context: &str, mut fetch: impl FnMut() -> Value) -> Value { + let deadline = Instant::now() + Duration::from_secs(15); + loop { + let payload = fetch(); + if !retryable_pre_admission_unavailable(&payload) { + return payload; + } + assert!( + Instant::now() < deadline, + "{context}: project open stayed retryable unavailable past the open grace: {payload}" + ); + std::thread::sleep(Duration::from_millis(50)); + } +} + +fn await_sdk_reset_problem( + context: &str, + client: &Client, + request: &::Request, +) -> (String, Value) { + let deadline = Instant::now() + Duration::from_secs(15); + loop { + let error = client + .execute::(request) + .expect_err("a refused store must not read as a healthy status"); + let (kind, envelope) = sdk_problem(error, context); + let payload = problem_envelope(&envelope, context); + if !retryable_pre_admission_unavailable(&payload) { + return (kind, envelope); + } + assert!( + Instant::now() < deadline, + "{context}: project open stayed retryable unavailable past the open grace: {payload}" + ); + std::thread::sleep(Duration::from_millis(50)); + } +} + /// Arms the daemon's one-shot fact-commit barrier, runs `request` on its own /// thread, holds the committed effect there until the request's own deadline /// has certainly expired, then releases it and returns what `request` produced. @@ -665,25 +721,23 @@ fn reset_required_survives_http_mcp_and_rust_sdk_across_restart() { "a typed HTTP terminal must not be reported as success: status {http_status}, body {http_body}" ); - let mcp_response = mcp_tool_call( - &home_path, - &project_path, - "tracedecay_storage_status", - &storage_status_body, - None, - ); - super::assert_reset_required( - &problem_envelope(&mcp_payload(&mcp_response), "MCP reset required"), - "MCP stdio host, first observation", - ); + let mcp_problem = await_settled_problem("MCP stdio host, first observation", || { + let mcp_response = mcp_tool_call( + &home_path, + &project_path, + "tracedecay_storage_status", + &storage_status_body, + None, + ); + problem_envelope(&mcp_payload(&mcp_response), "MCP reset required") + }); + super::assert_reset_required(&mcp_problem, "MCP stdio host, first observation"); let client = sdk_client(&mount, &identity); let request = serde_json::from_value(storage_status_body.clone()).expect("canonical storage status"); - let sdk_error = client - .execute::(&request) - .expect_err("a refused store must not read as a healthy status"); - let (sdk_kind, sdk_envelope) = sdk_problem(sdk_error, "Rust SDK reset required"); + let (sdk_kind, sdk_envelope) = + await_sdk_reset_problem("Rust SDK, first observation", &client, &request); assert_eq!( sdk_kind, "reset_required", "the Rust SDK must classify the terminal as reset required: {sdk_envelope}" @@ -708,36 +762,37 @@ fn reset_required_survives_http_mcp_and_rust_sdk_across_restart() { ); let mount = http_mount(&home_path); - let (_, http_body_after) = post_application( - &mount, - &identity, - STORAGE_STATUS_ROUTE, - &storage_status_body, - None, - ); - super::assert_reset_required( - &problem_envelope(&http_body_after, "HTTP reset required after restart"), - "HTTP mount, after a physical restart", - ); - - let mcp_after = mcp_tool_call( - &home_path, - &project_path, - "tracedecay_storage_status", - &storage_status_body, - None, - ); + let http_problem_after = await_settled_problem("HTTP mount, after a physical restart", || { + let (_, body) = post_application( + &mount, + &identity, + STORAGE_STATUS_ROUTE, + &storage_status_body, + None, + ); + problem_envelope(&body, "HTTP reset required after restart") + }); + super::assert_reset_required(&http_problem_after, "HTTP mount, after a physical restart"); + + let mcp_problem_after = + await_settled_problem("MCP stdio host, after a physical restart", || { + let mcp_after = mcp_tool_call( + &home_path, + &project_path, + "tracedecay_storage_status", + &storage_status_body, + None, + ); + problem_envelope(&mcp_payload(&mcp_after), "MCP reset required after restart") + }); super::assert_reset_required( - &problem_envelope(&mcp_payload(&mcp_after), "MCP reset required after restart"), + &mcp_problem_after, "MCP stdio host, after a physical restart", ); let client = sdk_client(&mount, &identity); - let sdk_error_after = client - .execute::(&request) - .expect_err("a refused store must not read as a healthy status after a restart"); let (sdk_kind_after, sdk_envelope_after) = - sdk_problem(sdk_error_after, "Rust SDK reset required after restart"); + await_sdk_reset_problem("Rust SDK reset required after restart", &client, &request); assert_eq!( sdk_kind_after, "reset_required", "the Rust SDK must keep classifying the terminal as reset required: {sdk_envelope_after}" From 0178593196f641a34d351595d57f3ef1d837bbbd Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 05:58:50 +0000 Subject: [PATCH 02/84] style(tests): collapse idle admission check The nested reconciling if trips clippy::collapsible_if under -D warnings. Co-authored-by: Zack Jackson --- .../code_index_ignored_dependencies_test/flight_tests.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs index 77309c5c22..feac73d938 100644 --- a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs +++ b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs @@ -172,10 +172,9 @@ async fn hold_idle_background_admission( loop { if registry.memory_stats().await.reconciling_worktrees == 0 && let Ok(permit) = admission.clone().try_acquire_owned() + && registry.memory_stats().await.reconciling_worktrees == 0 { - if registry.memory_stats().await.reconciling_worktrees == 0 { - return permit; - } + return permit; } assert!( std::time::Instant::now() <= deadline, From ba835fc173654f3be17190f4a1c3a8685367b7a4 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:11:14 +0000 Subject: [PATCH 03/84] test(advisory): accept peer-won Codex stop ingest The packaged host advisory journey required the Codex stop ingest to report committed. A project catch-up sweep that admits the rollout first reports exact_duplicate, which is the durable replay, and the assertion panicked. Accept both terminals, reject accepted_for_replay, and ride out the same typed warming or deferred progress the Cursor ingest already does. Co-authored-by: Zack Jackson --- .../advisory_runtime_acceptance.rs | 95 +++++++++++++------ 1 file changed, 64 insertions(+), 31 deletions(-) diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/advisory_runtime_acceptance.rs b/crates/tracedecay/tests/runtime_acceptance_suite/advisory_runtime_acceptance.rs index f397d6df9d..4f2d03dbd8 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/advisory_runtime_acceptance.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/advisory_runtime_acceptance.rs @@ -1088,37 +1088,70 @@ async fn packaged_host_ingest_delivers_a_registered_advisory_cycle() { "format": "json", }) .to_string(); - let stop_output = common::tracedecay_command_with_home(environment.home()) - .args([ - "tool", - "--project", - project_arg.as_str(), - "tracedecay_hook_runtime", - "--args", - stop_args.as_str(), - "--json", - ]) - .current_dir(&project) - .output() - .expect("invoke registered daemon stop path"); - assert!( - stop_output.status.success(), - "registered daemon stop ingest failed\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&stop_output.stdout), - String::from_utf8_lossy(&stop_output.stderr) - ); - let stop_response: Value = - serde_json::from_slice(&stop_output.stdout).expect("registered daemon stop response"); - let stop_payload: Value = serde_json::from_str( - stop_response["content"][0]["text"] - .as_str() - .expect("registered daemon stop response text"), - ) - .expect("registered daemon stop payload"); - assert_eq!( - stop_payload["status"], "committed", - "registered daemon stop ingest did not commit: {stop_response}" - ); + // The project catch-up sweep races this pass for the rollout just written. + // Admission is the durable commit. A sweep that admits it first leaves the + // hook with nothing new to persist and reports `exact_duplicate`. Both + // terminals prove the transcript is durable; `accepted_for_replay` proves + // neither. A deferred or still-warming pass is the same typed progress the + // Cursor ingest above rides out. + let stop_deadline = std::time::Instant::now() + Duration::from_secs(60); + loop { + let stop_output = common::tracedecay_command_with_home(environment.home()) + .args([ + "tool", + "--project", + project_arg.as_str(), + "tracedecay_hook_runtime", + "--args", + stop_args.as_str(), + "--json", + ]) + .current_dir(&project) + .output() + .expect("invoke registered daemon stop path"); + if stop_output.status.success() { + let stop_response: Value = serde_json::from_slice(&stop_output.stdout) + .expect("registered daemon stop response"); + let stop_payload: Value = serde_json::from_str( + stop_response["content"][0]["text"] + .as_str() + .expect("registered daemon stop response text"), + ) + .expect("registered daemon stop payload"); + if stop_payload["completed"] != false { + assert!( + matches!( + stop_payload["status"].as_str(), + Some("committed" | "exact_duplicate") + ), + "registered daemon stop ingest proved neither a commit nor a duplicate: {stop_response}\ndaemon log:\n{}", + std::fs::read_to_string(&daemon_log) + .expect("read isolated advisory daemon log"), + ); + break; + } + assert_eq!( + stop_payload["admission"]["retryable"], true, + "incomplete stop ingest must carry a retryable admission: {stop_response}" + ); + } else { + let stderr = String::from_utf8_lossy(&stop_output.stderr).into_owned(); + assert!( + stderr.contains("is warming in the background"), + "registered daemon stop ingest failed\nstdout:\n{}\nstderr:\n{stderr}\ndaemon log:\n{}", + String::from_utf8_lossy(&stop_output.stdout), + std::fs::read_to_string(&daemon_log).expect("read isolated advisory daemon log"), + ); + } + assert!( + std::time::Instant::now() < stop_deadline, + "registered daemon stop ingest did not complete before its deadline\nstdout:\n{}\nstderr:\n{}\ndaemon log:\n{}", + String::from_utf8_lossy(&stop_output.stdout), + String::from_utf8_lossy(&stop_output.stderr), + std::fs::read_to_string(&daemon_log).expect("read isolated advisory daemon log"), + ); + tokio::time::sleep(Duration::from_millis(250)).await; + } let advisory_args = json!({ // Serialized as a file URL rather than concatenated: a Windows native From 37d6bc0ecdae0b2ba6aa0eb0dd705a428eb9c003 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:12:45 +0000 Subject: [PATCH 04/84] test(advisory): wait out accepted_for_replay ingest A completed accepted_for_replay is not a durable commit. Keep polling until committed or exact_duplicate, or the deadline fails closed. Co-authored-by: Zack Jackson --- .../advisory_runtime_acceptance.rs | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/advisory_runtime_acceptance.rs b/crates/tracedecay/tests/runtime_acceptance_suite/advisory_runtime_acceptance.rs index f397d6df9d..d40e739a48 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/advisory_runtime_acceptance.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/advisory_runtime_acceptance.rs @@ -1005,13 +1005,25 @@ async fn packaged_host_ingest_delivers_a_registered_advisory_cycle() { .expect("registered daemon ingest response text"), ) .expect("registered daemon ingest payload"); - if payload["completed"] != false { + // `completed: true` with `accepted_for_replay` means the catch-up + // sweep has not yet drained this admission. That is not a durable + // commit, so keep polling until a terminal that proves the + // transcript, or the deadline reports the last payload. + if matches!( + payload["status"].as_str(), + Some("committed" | "exact_duplicate") + ) { break output; } - assert_eq!( - payload["admission"]["retryable"], true, - "incomplete ingest must carry a retryable admission: {response}" - ); + if payload["completed"] != false && payload["status"] != "accepted_for_replay" { + break output; + } + if payload["completed"] == false { + assert_eq!( + payload["admission"]["retryable"], true, + "incomplete ingest must carry a retryable admission: {response}" + ); + } } else { let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); assert!( From 2b6d8cf2e760efe99fc5ea514f1c1e86fce78592 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:16:17 +0000 Subject: [PATCH 05/84] fix(code-index): index clone postings by occurrence Resume verification looks up symbol_occurrence_id on WITHOUT ROWID posting tables whose keys start at class and language. That scan never finished, so a resumed successor could not return to ready. Opening a successor builds the occurrence indexes, including one staged before they existed, and the lookups require them. Co-authored-by: Zack Jackson --- .../projection/artifact/clone_successor.rs | 24 +++++++++++++++++-- .../candidate_producers.rs | 12 ++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_successor.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_successor.rs index 922ea3552b..4f67cb4680 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_successor.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_successor.rs @@ -80,6 +80,7 @@ impl CodeLexicalCloneSuccessorV1 { "clone successor does not match its prior artifact or metadata".to_owned(), )); } + ensure_clone_occurrence_indexes(&connection)?; Ok(Self { connection, mutation_gate, @@ -528,6 +529,25 @@ fn append_clone_fingerprints( Ok(()) } +/// `clone_exact_postings` and `clone_fingerprint_postings` are `WITHOUT ROWID` +/// tables whose primary keys start at `class` and `language`. Resume +/// verification looks up one `symbol_occurrence_id`, which that key cannot +/// serve. These secondary indexes are the lookup; without them each body +/// scans every posting already written and the successor never finishes. +const CLONE_OCCURRENCE_INDEXES_SQL: &str = "\ +CREATE INDEX IF NOT EXISTS clone_exact_postings_by_occurrence \ +ON clone_exact_postings(symbol_occurrence_id); +CREATE INDEX IF NOT EXISTS clone_fingerprint_postings_by_occurrence \ +ON clone_fingerprint_postings(symbol_occurrence_id);"; + +fn ensure_clone_occurrence_indexes( + connection: &Connection, +) -> Result<(), CodeLexicalArtifactErrorV1> { + connection + .execute_batch(CLONE_OCCURRENCE_INDEXES_SQL) + .map_err(sqlite_error) +} + fn verify_copied_source_page( connection: &Connection, page: &VerifiedSealedLexicalPageV1, @@ -633,7 +653,7 @@ fn verify_clone_page_rows( .collect::>(); let mut statement = connection .prepare( - "SELECT class, normalization_revision, digest, payload_digest FROM clone_exact_postings WHERE symbol_occurrence_id = ?1 ORDER BY class, normalization_revision, digest", + "SELECT class, normalization_revision, digest, payload_digest FROM clone_exact_postings INDEXED BY clone_exact_postings_by_occurrence WHERE symbol_occurrence_id = ?1 ORDER BY class, normalization_revision, digest", ) .map_err(sqlite_error)?; let stored_postings = statement @@ -681,7 +701,7 @@ fn verify_clone_fingerprint_page_rows( expected.sort(); let mut statement = connection .prepare( - "SELECT language, class, normalization_revision, fingerprint, token_position, payload_digest, body_digest FROM clone_fingerprint_postings WHERE symbol_occurrence_id = ?1 ORDER BY language, class, normalization_revision, fingerprint, token_position", + "SELECT language, class, normalization_revision, fingerprint, token_position, payload_digest, body_digest FROM clone_fingerprint_postings INDEXED BY clone_fingerprint_postings_by_occurrence WHERE symbol_occurrence_id = ?1 ORDER BY language, class, normalization_revision, fingerprint, token_position", ) .map_err(sqlite_error)?; let stored = statement diff --git a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs index e006138f83..07463f7adf 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs @@ -1453,6 +1453,15 @@ fn v16_clone_payloads_are_content_addressed_and_postings_page() { .append_page(&pages[0], &control) .expect("append first clone page"); drop(successor); + // A successor staged before occurrence indexes existed must still verify + // on resume. Dropping them here is that shipped shape. + rusqlite::Connection::open(&successor_path) + .expect("open successor before index backfill") + .execute_batch( + "DROP INDEX IF EXISTS clone_exact_postings_by_occurrence; + DROP INDEX IF EXISTS clone_fingerprint_postings_by_occurrence;", + ) + .expect("drop occurrence indexes"); let mut successor = CodeLexicalCloneSuccessorV1::open_or_create( &legacy_path, &successor_path, @@ -1461,6 +1470,9 @@ fn v16_clone_payloads_are_content_addressed_and_postings_page() { CODE_LEXICAL_ARTIFACT_BUILD_MEMORY_BUDGET_BYTES_V1, ) .expect("resume clone-only successor"); + successor + .verify_resumed_page(&pages[0], &control) + .expect("resumed clone page verifies through the occurrence index"); assert_eq!( successor .next_cursor() From 787892cab79d01b2541a0b69e11d7e3dda14e9bb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:19:02 +0000 Subject: [PATCH 06/84] fix(code-index): keep directory pointer faults in publication family A directory in the active pointer slot makes read and rename return EISDIR. That OS error was filed as unavailability, so a coalesced publication failure left the scheduler publication family. Refuse the non-file slot as reset-required corruption before either syscall. Co-authored-by: Zack Jackson --- .../src/code_index_generations.rs | 22 ++++- .../code_index_scheduler/publication_store.rs | 39 ++++++++- .../flight_tests.rs | 81 ++++++++++++++----- 3 files changed, 116 insertions(+), 26 deletions(-) diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations.rs b/crates/tracedecay-code-index-retention/src/code_index_generations.rs index b4976c6602..69bfce445e 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations.rs @@ -1745,7 +1745,27 @@ fn read_active_pointer( store_root: &Path, ) -> Result { let path = store_root.join(ACTIVE_POINTER_FILE); - let bytes = std::fs::read(&path).map_err(storage)?; + // A directory in the pointer slot makes `read(2)` return EISDIR. That is + // the same corrupt authority the publication store refuses; do not let the + // OS error replace the typed unsafe-state. + match std::fs::metadata(&path) { + Ok(metadata) if metadata.file_type().is_file() => {} + Ok(_) => { + return Err(CodeGenerationRetentionErrorV1::UnsafeState( + "active code-generation pointer is not a regular file".to_owned(), + )); + } + Err(error) => return Err(storage(error)), + } + let bytes = std::fs::read(&path).map_err(|error| { + if error.kind() == std::io::ErrorKind::IsADirectory { + CodeGenerationRetentionErrorV1::UnsafeState( + "active code-generation pointer is not a regular file".to_owned(), + ) + } else { + storage(error) + } + })?; serde_json::from_slice(&bytes).map_err(|error| { CodeGenerationRetentionErrorV1::UnsafeState(format!( "active pointer '{}' is corrupt: {error}", diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs index 3422489f5e..636f838d89 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs @@ -871,6 +871,33 @@ impl DaemonCodeIndexPublicationStoreV1 { CodeIndexPublicationStoreErrorV1::CorruptionResetRequired(error.to_string()) } + /// A pointer slot that is not a regular file is a corrupt authority. + /// + /// `read(2)` and `rename(2)` both report that shape as `EISDIR`. Mapping + /// the OS error to `Unavailable` (or letting it surface as a raw I/O + /// fault) misclassifies a broken publication pointer. Callers in the + /// scheduler publication family must see reset-required corruption. + fn corrupt_non_file_pointer() -> CodeIndexPublicationStoreErrorV1 { + Self::corruption("active code-generation pointer is not a regular file") + } + + fn map_pointer_io(error: std::io::Error) -> CodeIndexPublicationStoreErrorV1 { + if error.kind() == std::io::ErrorKind::IsADirectory { + Self::corrupt_non_file_pointer() + } else { + Self::unavailable(error) + } + } + + fn require_regular_pointer_slot(&self) -> Result<(), CodeIndexPublicationStoreErrorV1> { + match std::fs::metadata(&self.active_path) { + Ok(metadata) if metadata.file_type().is_file() => Ok(()), + Ok(_) => Err(Self::corrupt_non_file_pointer()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(Self::map_pointer_io(error)), + } + } + fn acquire_generation_read_lock( &self, ) -> Result { @@ -1089,8 +1116,11 @@ impl DaemonCodeIndexPublicationStoreV1 { .unwrap_or_else(PoisonError::into_inner) = None; return Ok(None); } - Err(error) => return Err(Self::unavailable(error)), + Err(error) => return Err(Self::map_pointer_io(error)), }; + if !metadata.file_type().is_file() { + return Err(Self::corrupt_non_file_pointer()); + } if metadata.len() > MAX_DURABLE_PUBLICATION_POINTER_BYTES { return Err(Self::corruption( "durable code-generation index exceeds its byte bound", @@ -1102,7 +1132,7 @@ impl DaemonCodeIndexPublicationStoreV1 { // a fixed-width pointer through another path, and a 1-second mtime // filesystem can leave both unchanged while the bytes move. The memo // is reused only when the file digest matches. - let bytes = std::fs::read(&self.active_path).map_err(Self::unavailable)?; + let bytes = std::fs::read(&self.active_path).map_err(Self::map_pointer_io)?; let digest = Self::state_digest(&bytes); { let mut memo = self @@ -2558,8 +2588,11 @@ impl CodeIndexAtomicPublicationPort for DaemonCodeIndexPublicationStoreV1 { std::fs::remove_file(&temporary).map_err(Self::unavailable)?; } hotpath::measure_block!("code_index.generation.publish.pointer_commit", { + // Refuse a directory (or any non-file) before `rename(2)`. Replacing + // one returns EISDIR, which is not a publication-family fault. + self.require_regular_pointer_slot()?; Self::write_durable(&temporary, &bytes)?; - std::fs::rename(&temporary, &self.active_path).map_err(Self::unavailable)?; + std::fs::rename(&temporary, &self.active_path).map_err(Self::map_pointer_io)?; Self::sync_directory( self.active_path .parent() diff --git a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs index bc1388df7f..e4c972bc6b 100644 --- a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs +++ b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs @@ -1,7 +1,10 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Condvar, Mutex}; -use tracedecay_code_index::production::CodeIndexProductionErrorV1; +use tracedecay_code_index::production::{ + CodeIndexProductionErrorV1, CodeIndexPublicationStoreErrorV1, +}; +use tracedecay_code_index_retention::code_index_generations::try_acquire_code_generation_store_lock; use super::*; @@ -148,15 +151,50 @@ export function GenerationAnchor(value: PublicWidget) { return value; } } fn assert_publication_error(error: CodeIndexSchedulerErrorV1) { + let CodeIndexSchedulerErrorV1::Production(CodeIndexProductionErrorV1::Publication( + CodeIndexPublicationStoreErrorV1::CorruptionResetRequired(detail), + )) = &error + else { + panic!( + "coalesced failure must stay in the scheduler publication family, not an EISDIR misclass: {error:?}" + ); + }; + assert!( + detail.contains("not a regular file"), + "a directory pointer slot is publication corruption, got {detail}" + ); assert!( - matches!( - error, - CodeIndexSchedulerErrorV1::Production(CodeIndexProductionErrorV1::Publication(_)) - ), - "coalesced failure must preserve the production publication error family" + !detail.contains("Is a directory") && !detail.contains("os error 21"), + "publication corruption must not carry the raw EISDIR OS error: {detail}" ); } +/// Hold the only background permit once no pass is in flight. +/// +/// Text seating keeps `reconcile_in_progress` after it drops the scheduler +/// mutex, and that pass can still rename a valid active pointer. A truncated +/// pointer written in that window is not a closed fault. Occupying the permit +/// while the owner has not entered its pass stops that rewrite. +async fn hold_idle_background_admission( + registry: &CodeIndexSchedulerRegistryV1, +) -> tokio::sync::OwnedSemaphorePermit { + let admission = registry.background_reconcile_admission(); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + if registry.memory_stats().await.reconciling_worktrees == 0 + && let Ok(permit) = admission.clone().try_acquire_owned() + && registry.memory_stats().await.reconciling_worktrees == 0 + { + return permit; + } + assert!( + std::time::Instant::now() <= deadline, + "background reconcile did not go idle before publication fault injection" + ); + tokio::time::sleep(Duration::from_millis(2)).await; + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn aborted_flight_owner_wakes_follower_and_allows_a_fresh_owner() { let fixture = fixture(); @@ -242,6 +280,8 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { let registry = Arc::new(mount(fixture.path(), &store, 1).await); let baseline = latest(®istry, fixture.path()).await; let request = request_for(&baseline, "pkg"); + let idle_admission = hold_idle_background_admission(®istry).await; + registry.clear_pending_wake_for_scope(&request.scope).await; let hold = SchedulerHold::acquire(®istry, fixture.path()).await; let (owner_control, owner_entered) = BlockingNthControl::new(4); @@ -283,22 +323,12 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { &fixture.path().canonicalize().expect("canonical fixture"), ); let pointer_path = scoped_store.join("active-code-generation-v1.json"); - // Every production writer of the active pointer reads it, edits it in - // memory and renames a temporary over it while holding the exclusive - // generation-store lock. Corrupting the file without that lock races an - // in-flight read-modify-write whose rename then restores a valid pointer, - // and this owner publishes instead of failing closed. The racer is the - // background pass tail: it releases the background admission permit this - // owner then takes (registry/mount.rs, "release the background admission - // permit before HeadOpening / graph work") and keeps attaching the - // generation's text artifact afterwards, so neither the held admission - // nor the held scheduler mutex proves the store is quiet. Taking the - // store lock does: being granted it means no writer is mid-transaction, - // and any writer that starts after it is released reads the corruption - // under the lock and refuses instead of overwriting it. + // Writers rename a temporary over the active pointer while holding the + // generation-store lock. A truncated file is not a closed fault: a pass + // that already read a valid pointer can rename it back. A directory cannot + // be renamed over. Taking the lock first means no writer is mid-transaction + // when the slot stops being a regular file. let pointer_bytes = { - use tracedecay_code_index_retention::code_index_generations::try_acquire_code_generation_store_lock; - let store_lock = tokio::time::timeout(Duration::from_secs(5), async { loop { if let Some(lock) = try_acquire_code_generation_store_lock(&scoped_store) @@ -312,10 +342,16 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { .await .expect("no generation-store writer is mid-transaction"); let pointer_bytes = std::fs::read(&pointer_path).expect("read active pointer"); - std::fs::write(&pointer_path, b"{").expect("corrupt active pointer"); + // `rename(2)` replaces a truncated file with a valid pointer. A + // directory cannot be renamed over, so the fault stays closed. The + // scheduler must report publication corruption, not the EISDIR that + // read and rename return for that directory. + std::fs::remove_file(&pointer_path).expect("remove active pointer"); + std::fs::create_dir(&pointer_path).expect("replace active pointer with a directory"); drop(store_lock); pointer_bytes }; + drop(idle_admission); owner_control.release(); hold.release(); @@ -332,6 +368,7 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { assert_publication_error(owner_error); assert_publication_error(follower_error); + std::fs::remove_dir_all(&pointer_path).expect("remove faulted pointer node"); std::fs::write(pointer_path, pointer_bytes).expect("restore active pointer"); registry.shutdown().await; } From a943654d99f0ed61a4f8b15aff278562a41fbeb7 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:19:15 +0000 Subject: [PATCH 07/84] fix(ci): lint integration commits on dispatch workflow_dispatch admits an integration branch and previously skipped commitlint, so batch-fold follow-ups with headers of 73, 75, and 85 characters landed in #1796 and failed the master push. Dispatch now lints the commits a merge onto the default branch would introduce. header-max-length stays 72. A dispatch of the default branch does not rejudge published history. Co-authored-by: Zack Jackson --- .github/workflows/ci.yml | 23 ++--- scripts/lint-ci-commits.sh | 85 ++++++++++++++++++ scripts/test-lint-commit-range.py | 143 ++++++++++++++++++++++++++++++ 3 files changed, 236 insertions(+), 15 deletions(-) create mode 100755 scripts/lint-ci-commits.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a399aad93..c8f6527c69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -219,24 +219,17 @@ jobs: python3 scripts/test-check-dev-skill-mirrors.py python3 scripts/check-dev-skill-mirrors.py check - - name: Validate pushed commit messages - if: github.event_name == 'push' + # Dispatch admits an integration branch. Judge the commits a merge onto + # the default branch would introduce, with the same linter a push uses. + # A push still uses the before SHA, so published history is not rejudged. + - name: Validate commit messages + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' env: + EVENT_NAME: ${{ github.event_name }} BEFORE_SHA: ${{ github.event.before }} HEAD_SHA: ${{ github.sha }} - run: | - set -euo pipefail - if [ "$BEFORE_SHA" = "0000000000000000000000000000000000000000" ]; then - if git rev-parse "${HEAD_SHA}^" >/dev/null 2>&1; then - base_sha="${HEAD_SHA}^" - else - git show --no-patch --format=%B "$HEAD_SHA" | npm run lint:commit -- - exit 0 - fi - else - base_sha="$BEFORE_SHA" - fi - node scripts/lint-commit-range.mjs --repository "$PWD" "$base_sha" "$HEAD_SHA" + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: scripts/lint-ci-commits.sh macos-test-partition: name: Test macOS ${{ matrix.group }} diff --git a/scripts/lint-ci-commits.sh b/scripts/lint-ci-commits.sh new file mode 100755 index 0000000000..64a149863f --- /dev/null +++ b/scripts/lint-ci-commits.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Lint the commits a CI event is admitting. +# +# A master push lints github.event.before..HEAD, which is how an integration +# merge is judged after it lands. workflow_dispatch is the admission path for +# that integration branch, so it lints the same not-yet-on-the-default-branch +# range. Already published history is not rejudged: a dispatch of the default +# branch has an empty range. +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +project_root=$(cd "${script_dir}/.." && pwd) +repository=${REPOSITORY:-$project_root} +event=${EVENT_NAME:-} +head=${HEAD_SHA:-} +zero_sha=0000000000000000000000000000000000000000 + +if [[ -z $event || -z $head ]]; then + echo "usage: EVENT_NAME= HEAD_SHA= [BEFORE_SHA=] [DEFAULT_BRANCH=] [REPOSITORY=] $0" >&2 + exit 2 +fi + +lint_range() { + local base=$1 + node "${script_dir}/lint-commit-range.mjs" --repository "$repository" "$base" "$head" +} + +lint_root_commit() { + git -C "$repository" show --no-patch --format=%B "$head" | ( + cd "$project_root" + npm run --silent lint:commit -- + ) +} + +resolve_default_branch() { + local branch=$1 + local remote_ref="refs/remotes/origin/${branch}" + if git -C "$repository" remote get-url origin >/dev/null 2>&1; then + git -C "$repository" fetch --no-tags --quiet origin \ + "+refs/heads/${branch}:${remote_ref}" >/dev/null + fi + if git -C "$repository" rev-parse --verify --quiet "$remote_ref" >/dev/null; then + echo "$remote_ref" + return 0 + fi + if git -C "$repository" rev-parse --verify --quiet "refs/heads/${branch}" >/dev/null; then + echo "refs/heads/${branch}" + return 0 + fi + echo "commit lint: default branch ${branch} is not available" >&2 + return 1 +} + +case "$event" in + push) + before=${BEFORE_SHA:-} + if [[ -z $before ]]; then + echo "commit lint: push requires BEFORE_SHA" >&2 + exit 2 + fi + if [[ $before == "$zero_sha" ]]; then + if git -C "$repository" rev-parse --verify --quiet "${head}^" >/dev/null; then + lint_range "${head}^" + else + lint_root_commit + fi + else + lint_range "$before" + fi + ;; + workflow_dispatch) + default_branch=${DEFAULT_BRANCH:-} + if [[ -z $default_branch ]]; then + echo "commit lint: workflow_dispatch requires DEFAULT_BRANCH" >&2 + exit 2 + fi + upstream=$(resolve_default_branch "$default_branch") + base=$(git -C "$repository" merge-base "$head" "$upstream") + lint_range "$base" + ;; + *) + echo "commit lint: unsupported event ${event}" >&2 + exit 2 + ;; +esac diff --git a/scripts/test-lint-commit-range.py b/scripts/test-lint-commit-range.py index aac5532fd1..f58dd0f965 100755 --- a/scripts/test-lint-commit-range.py +++ b/scripts/test-lint-commit-range.py @@ -14,6 +14,21 @@ REPOSITORY_ROOT = Path(__file__).resolve().parent.parent LINT_RANGE = REPOSITORY_ROOT / "scripts" / "lint-commit-range.mjs" +LINT_CI = REPOSITORY_ROOT / "scripts" / "lint-ci-commits.sh" + +# Subjects from the integration fold that dispatch admitted and the following +# master push rejected. Each matches the typed-header grammar and fails only +# because the header is longer than the configured maximum. +BATCH_FOLLOWUP_SUBJECTS = ( + "fix(pr-1633): warm the diagnose fixture through the shared support helper", + "fix(pr-1740): resolve git through common::git_program and sort the mod line", + "fix(pr-1617): share the exact-arguments dispatch instead of widening CaptureTransport", +) +HYGIENIC_FOLLOWUP_MESSAGES = ( + "fix(pr-1633): warm the diagnose fixture through shared support\n\nhelper", + "fix(pr-1740): resolve git through common::git_program and sort mods\n\nthe mod line", + "fix(pr-1617): share exact-argument dispatch without widening transport\n\ninstead of widening CaptureTransport", +) def run( @@ -155,6 +170,134 @@ def test_node_startup_count_is_constant_for_a_large_range(self) -> None: f"elapsed_ms={elapsed_ms}" ) + def lint_ci( + self, + *, + event: str, + head: str, + before: str | None = None, + default_branch: str | None = None, + ) -> subprocess.CompletedProcess[str]: + environment = os.environ.copy() + environment.update( + { + "EVENT_NAME": event, + "HEAD_SHA": head, + "REPOSITORY": str(self.root), + } + ) + if before is not None: + environment["BEFORE_SHA"] = before + if default_branch is not None: + environment["DEFAULT_BRANCH"] = default_branch + return run( + ["bash", str(LINT_CI)], + cwd=self.root, + env=environment, + check=False, + ) + + def test_dispatch_rejects_batch_followup_headers_over_the_maximum(self) -> None: + base = self.commit("chore(test): establish fixture base") + master = self.commit("fix(test): keep the default branch valid", base) + run(["git", "branch", "master", master], cwd=self.root) + head = master + followups = [] + for subject in BATCH_FOLLOWUP_SUBJECTS: + self.assertGreater(len(subject), 72) + head = self.commit(subject, head) + followups.append(head) + + result = self.lint_ci( + event="workflow_dispatch", + head=head, + default_branch="master", + ) + output = result.stdout + result.stderr + + self.assertNotEqual(result.returncode, 0, output) + for sha, subject in zip(followups, BATCH_FOLLOWUP_SUBJECTS, strict=True): + self.assertIn(sha, output) + self.assertIn(subject, output) + self.assertIn("header-max-length", output) + self.assertNotIn(master, output) + + def test_dispatch_accepts_the_same_followups_once_the_header_fits(self) -> None: + base = self.commit("chore(test): establish fixture base") + master = self.commit("fix(test): keep the default branch valid", base) + run(["git", "branch", "master", master], cwd=self.root) + head = master + for message in HYGIENIC_FOLLOWUP_MESSAGES: + header = message.split("\n", 1)[0] + self.assertLessEqual(len(header), 72) + self.assertTrue(header.startswith("fix(pr-")) + head = self.commit(message, head) + + result = self.lint_ci( + event="workflow_dispatch", + head=head, + default_branch="master", + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_dispatch_of_the_default_branch_does_not_rejudge_published_history(self) -> None: + base = self.commit("chore(test): establish fixture base") + published = self.commit(BATCH_FOLLOWUP_SUBJECTS[0], base) + master = self.commit("fix(test): keep the default branch valid", published) + run(["git", "branch", "master", master], cwd=self.root) + + result = self.lint_ci( + event="workflow_dispatch", + head=master, + default_branch="master", + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn(published, result.stdout + result.stderr) + + def test_push_still_lints_the_before_sha_range(self) -> None: + base = self.commit("chore(test): establish fixture base") + head = self.commit(BATCH_FOLLOWUP_SUBJECTS[2], base) + + result = self.lint_ci(event="push", head=head, before=base) + output = result.stdout + result.stderr + + self.assertNotEqual(result.returncode, 0, output) + self.assertIn(head, output) + self.assertIn("header-max-length", output) + + def test_push_of_a_root_commit_lints_that_message(self) -> None: + valid = self.commit("chore(test): establish fixture base") + invalid = self.commit("not a conventional header") + + valid_result = self.lint_ci( + event="push", + head=valid, + before="0000000000000000000000000000000000000000", + ) + invalid_result = self.lint_ci( + event="push", + head=invalid, + before="0000000000000000000000000000000000000000", + ) + + self.assertEqual( + valid_result.returncode, + 0, + valid_result.stdout + valid_result.stderr, + ) + self.assertNotEqual(invalid_result.returncode, 0) + self.assertIn("type-empty", invalid_result.stdout + invalid_result.stderr) + + def test_unsupported_event_is_rejected(self) -> None: + head = self.commit("chore(test): establish fixture base") + + result = self.lint_ci(event="schedule", head=head) + + self.assertEqual(result.returncode, 2) + self.assertIn("unsupported event", result.stderr) + if __name__ == "__main__": unittest.main() From a0d26c7050eee31638b940630a392a6ec5c9bf99 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 06:24:56 +0000 Subject: [PATCH 08/84] test(daemon): prove the protected preview stays redacted `tracedecay_configuration_protected_preview` is the dry run a host calls before it commits a protected setting, and nothing covered its host-facing contract end to end. Drive the tool through MCP `tools/call` on the production composition and assert what a caller actually receives: a plan bound to the supplied revision, the redacted per-setting digests, the operation digest echoed as the preview digest, the five-minute validity window, and a preview id the host can apply. The rule identity and the denied capability must never appear in the rendered answer, a stale revision must be a typed `configuration.conflict`, an empty capability set a typed `configuration.invalid_request`, and no preview may advance the configuration revision. Co-Authored-By: Claude Fable 5.1 --- .../src/daemon/production_harness.rs | 3 + ...guration_protected_preview_journey_test.rs | 275 ++++++++++++++++++ 2 files changed, 278 insertions(+) create mode 100644 crates/tracedecay/src/daemon/production_harness/configuration_protected_preview_journey_test.rs diff --git a/crates/tracedecay/src/daemon/production_harness.rs b/crates/tracedecay/src/daemon/production_harness.rs index 1f2c6daaaf..a0ffb46bf6 100644 --- a/crates/tracedecay/src/daemon/production_harness.rs +++ b/crates/tracedecay/src/daemon/production_harness.rs @@ -1535,6 +1535,9 @@ mod generation_retention_test; #[cfg(test)] mod configuration_idempotency_journey_test; +#[cfg(test)] +mod configuration_protected_preview_journey_test; + #[cfg(test)] mod read_only_project_open_journey_test; diff --git a/crates/tracedecay/src/daemon/production_harness/configuration_protected_preview_journey_test.rs b/crates/tracedecay/src/daemon/production_harness/configuration_protected_preview_journey_test.rs new file mode 100644 index 0000000000..74b2191aa4 --- /dev/null +++ b/crates/tracedecay/src/daemon/production_harness/configuration_protected_preview_journey_test.rs @@ -0,0 +1,275 @@ +//! Host-facing behavior of `tracedecay_configuration_protected_preview`. +//! +//! The tool is a dry-run: the answer is a redacted plan bound to the revision +//! the caller supplied, and a wrong revision or an invalid change is a typed +//! problem rather than a committed setting. Callers observe that through MCP +//! `tools/call`, which is the path this journey drives. + +use std::collections::BTreeSet; +use std::path::Path; + +use serde_json::{Value, json}; +use tempfile::TempDir; +use tracedecay_contracts::ConfigurationProtectedPreviewRequestV1; +use tracedecay_domain::configuration::{ + AccessRuleId, AuthorityRef, ConfigurationRevisionId, ProtectedChange, RuleEffect, + ScopeAccessRule, ScopeAccessSubjectV1, SourceBindingId, SourceKindV1, +}; +use tracedecay_domain::{CapabilityId, ManifestDigest}; + +use super::journey_test_support::{git, tool_answer}; +use super::*; + +const ACCESS_RULE_ID: &str = "access-rule.preview-cursor-deny"; +const DENIED_CAPABILITY: &str = "capability.work.generate_proposal"; +const ABSENT_BINDING_ID: &str = "source-binding.preview-absent"; +const STALE_REVISION: &str = "configuration.revision.protected-preview-not-current"; + +fn initialize_project(project: &Path) { + std::fs::create_dir_all(project.join("src")).expect("project source"); + std::fs::write(project.join("src/lib.rs"), "pub fn preview_probe() {}\n") + .expect("project source file"); + git(project, &["init", "--quiet"]); +} + +fn preview_arguments(change: &ProtectedChange, revision: &ConfigurationRevisionId) -> Value { + let mut arguments = serde_json::to_value(ConfigurationProtectedPreviewRequestV1 { + change: change.clone(), + expected_revision: revision.clone(), + }) + .expect("protected preview arguments"); + arguments["format"] = json!("json"); + arguments +} + +fn deny_cursor_work(project_id: tracedecay_domain::ProjectId) -> ProtectedChange { + ProtectedChange::UpsertAccessRule( + ScopeAccessRule::new( + AccessRuleId::new(ACCESS_RULE_ID).expect("access rule identity"), + ScopeAccessSubjectV1 { + actor: None, + operation: None, + source_kind: Some(SourceKindV1::Cursor), + }, + AuthorityRef::Project(project_id), + BTreeSet::from([ + CapabilityId::new(DENIED_CAPABILITY).expect("generate proposal capability") + ]), + RuleEffect::Deny, + None, + ) + .expect("deny-only work rule"), + ) +} + +async fn call_preview( + harness: &ProductionProjectCompositionHarnessV1, + project: &Path, + arguments: Value, +) -> (bool, Value) { + let response = harness + .call_tool( + project, + "tracedecay_configuration_protected_preview", + arguments, + ) + .await + .expect("protected preview tools/call"); + tool_answer(&response) +} + +fn assert_redacted_plan( + payload: &Value, + revision: &str, + setting_key: &str, + operation: &str, + before_digest: &str, + after_digest: &str, + hidden: &[&str], +) { + assert_eq!(payload["outcome"]["outcome"], "preview"); + assert_eq!( + payload["outcome"]["value"]["effect_class"], + "configuration_write" + ); + let plan = &payload["outcome"]["value"]["payload"]; + assert_eq!(plan["base_revision_id"], revision); + assert_eq!( + plan["redacted_changes"], + json!([{ + "setting_key": setting_key, + "operation": operation, + "before_digest": before_digest, + "after_digest": after_digest, + }]) + ); + assert_eq!(plan["operation_digest"], after_digest); + assert_eq!(payload["outcome"]["value"]["preview_digest"], after_digest); + assert_eq!( + payload["outcome"]["value"]["preview_id"], plan["plan_id"], + "the preview id the host applies is the plan id" + ); + let plan_id = plan["plan_id"].as_str().expect("plan id"); + assert!( + plan_id.starts_with("configuration.plan.v1."), + "plan id {plan_id} is not a configuration plan" + ); + let created_at = plan["created_at"].as_i64().expect("plan created_at"); + let expires_at = plan["expires_at"].as_i64().expect("plan expires_at"); + assert_eq!( + expires_at - created_at, + 300_000_000, + "a protected preview stays valid for five minutes" + ); + let rendered = serde_json::to_string(payload).expect("preview json"); + for secret in hidden { + assert!( + !rendered.contains(secret), + "preview leaked {secret}: {rendered}" + ); + } +} + +fn assert_problem( + payload: &Value, + kind: &str, + code: &str, + message: &str, + retry: &str, + legal_actions: Value, +) { + assert_eq!(payload["problem"]["kind"], kind, "{payload}"); + assert_eq!(payload["problem"]["code"], code, "{payload}"); + assert_eq!(payload["problem"]["message"], message, "{payload}"); + assert_eq!(payload["problem"]["diagnostic"]["code"], code, "{payload}"); + assert_eq!( + payload["problem"]["diagnostic"]["message"], message, + "{payload}" + ); + assert_eq!(payload["problem"]["retry"], retry, "{payload}"); + assert_eq!( + payload["problem"]["legal_actions"], legal_actions, + "{payload}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn protected_preview_redacts_the_change_and_refuses_stale_or_invalid_input() { + let isolation = TempDir::new().expect("journey isolation"); + let project = isolation.path().join("project"); + initialize_project(&project); + + let harness = ProductionProjectCompositionHarnessV1::open(isolation.path(), [project.clone()]) + .await + .expect("production composition"); + let graph = harness.server(&project).expect("project server").cg().await; + let project_id = graph + .configuration_runtime() + .configuration_target() + .project_id + .clone(); + let current = graph + .configuration_runtime() + .client() + .current() + .await + .expect("current configuration"); + let revision = current.revision_id().clone(); + let before_digest: ManifestDigest = current.snapshot().effective_behavior_digest.clone(); + drop(graph); + + let access_rule = deny_cursor_work(project_id.clone()); + let access_digest = access_rule + .compute_digest() + .expect("access rule digest") + .as_str() + .to_owned(); + let (refused, accepted) = call_preview( + &harness, + &project, + preview_arguments(&access_rule, &revision), + ) + .await; + assert!(!refused, "access-rule preview was refused: {accepted}"); + assert_redacted_plan( + &accepted, + revision.as_str(), + "scope.access_rules.v1", + "access_rule_upsert", + before_digest.as_str(), + &access_digest, + &[ACCESS_RULE_ID, DENIED_CAPABILITY], + ); + + let unbind = ProtectedChange::UnbindSource { + binding_id: SourceBindingId::new(ABSENT_BINDING_ID).expect("binding identity"), + }; + let unbind_digest = unbind + .compute_digest() + .expect("unbind digest") + .as_str() + .to_owned(); + assert_ne!( + access_digest, unbind_digest, + "the two submitted changes must not share a digest" + ); + let (refused, unbound) = + call_preview(&harness, &project, preview_arguments(&unbind, &revision)).await; + assert!(!refused, "unbind preview was refused: {unbound}"); + assert_redacted_plan( + &unbound, + revision.as_str(), + "scope.source_bindings.v1", + "source_unbind", + before_digest.as_str(), + &unbind_digest, + &[ABSENT_BINDING_ID], + ); + + let mut stale = preview_arguments(&access_rule, &revision); + stale["expected_revision"] = json!(STALE_REVISION); + let (refused, conflict) = call_preview(&harness, &project, stale).await; + assert!(refused, "a stale revision must be a tool error: {conflict}"); + assert_problem( + &conflict, + "conflict", + "configuration.conflict", + "The configuration request conflicts with current state", + "after_revalidate", + json!(["refresh"]), + ); + + let mut invalid = preview_arguments(&access_rule, &revision); + invalid["change"]["value"]["capabilities"] = json!([]); + let (refused, rejected) = call_preview(&harness, &project, invalid).await; + assert!( + refused, + "an empty capability set must be a tool error: {rejected}" + ); + assert_problem( + &rejected, + "invalid_request", + "configuration.invalid_request", + "The configuration request is invalid: access rule capabilities must not be empty", + "never", + json!([]), + ); + + let graph = harness.server(&project).expect("project server").cg().await; + let unchanged = graph + .configuration_runtime() + .client() + .current() + .await + .expect("configuration after previews") + .revision_id() + .clone(); + drop(graph); + assert_eq!( + unchanged.as_str(), + revision.as_str(), + "protected preview must not commit a revision" + ); + + harness.shutdown().await; +} From 74459e4b026495e38b83057ac0c8930ad9e9c174 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:25:25 +0000 Subject: [PATCH 09/84] fix(build): keep eval lexical projection off transport The root crate depended on tracedecay-search-eval unconditionally, and cargo unifies that package's tracedecay-query/search-eval feature into every test target. Transport suites therefore compiled the eval-only in-memory lexical projection. The dependency is now opt-in and enabled only by the journeys that compare the CLI receipt to the library. Co-authored-by: Zack Jackson --- .github/linux-test-partitions.json | 3 ++- .github/workflows/ci.yml | 12 ++++++++---- crates/tracedecay-query/Cargo.toml | 2 ++ crates/tracedecay/Cargo.toml | 13 ++++++++++++- .../tracedecay/tests/product_surface_suite/main.rs | 3 +++ .../tests/runtime_acceptance_suite/main.rs | 4 ++++ 6 files changed, 31 insertions(+), 6 deletions(-) diff --git a/.github/linux-test-partitions.json b/.github/linux-test-partitions.json index 9febd5785a..1b5b8aecd9 100644 --- a/.github/linux-test-partitions.json +++ b/.github/linux-test-partitions.json @@ -68,7 +68,8 @@ "example:tracedecay-host-cli-fixture" ], "features": [ - "tracedecay/test-helpers" + "tracedecay/test-helpers", + "tracedecay/search-eval" ] }, { diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a399aad93..004961420a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -431,10 +431,14 @@ jobs: # `--lib` / `--test ` / `--bins` decide what compiles, and a filterset # would compile everything and skip at run time. The three root partitions # share one package selection (`tracedecay`, `tracedecay-cli`, - # `tracedecay-search-eval`) with the root fixture feature, so they resolve - # one identical dependency graph and every cargo invocation inside a job - # (the test build, the executables the suites spawn) is a cache hit against - # it. `scripts/linux-test-partitions.py check` proves, from `cargo + # `tracedecay-search-eval`) with the root fixture feature. Journeys also + # enable `tracedecay/search-eval`, the only link from the root crate to the + # evaluator library. It stays off every other partition: an unconditional + # dependency unifies `tracedecay-query/search-eval` into every test + # target of the package, and the transport suites then compile the eval-only + # lexical projection. Every cargo invocation inside a job (the test build, + # the executables the suites spawn) is a cache hit against that job's + # resolution. `scripts/linux-test-partitions.py check` proves, from `cargo # metadata`, that every test target in the workspace is selected by exactly # one partition or listed under `not_run` with a reason, so a new crate or # suite cannot fall out of the lane silently; `scope-gate` derives the diff --git a/crates/tracedecay-query/Cargo.toml b/crates/tracedecay-query/Cargo.toml index 015e41e6cb..ffc65866d4 100644 --- a/crates/tracedecay-query/Cargo.toml +++ b/crates/tracedecay-query/Cargo.toml @@ -47,6 +47,8 @@ test-helpers = ["tracedecay-temporal-query/test-helpers"] # CodeLexicalProjectionAdapterV1` and its builder). Production retrieval reads # the durable lexical artifact; only the search-quality evaluator and this # crate's own suites build projections directly from admitted chunks. +# `tracedecay` must not depend on the evaluator unconditionally: that unifies +# this feature into every root test target, including transport. search-eval = [] # The grammar tier the shipped product indexes with, forwarded exactly as # `tracedecay-cli`'s `production` forwards `tracedecay/production`. Only the diff --git a/crates/tracedecay/Cargo.toml b/crates/tracedecay/Cargo.toml index 2a7f1cf535..a6616a03e4 100644 --- a/crates/tracedecay/Cargo.toml +++ b/crates/tracedecay/Cargo.toml @@ -231,6 +231,14 @@ test-transport = [ "tracedecay-code-index-runtime/test-transport", ] +# The evaluator library is the only selector of `tracedecay-query/search-eval`, +# the eval-only in-memory lexical projection. Cargo rejects optional +# dev-dependencies, and an unconditional one unifies this feature into every +# test target, so the transport suites compile that projection. Journeys that +# compare the CLI receipt to the library enable this feature. `test-transport` +# and `production` must not. +search-eval = ["dep:tracedecay-search-eval"] + # The typed RMCP benchmark is the only consumer of the client-side RMCP # transport. Keep that surface out of ordinary integration-fixture builds: # the benchmark enables its complete client/runtime dependency set explicitly. @@ -340,6 +348,10 @@ keyring = { version = "4.1.5", features = ["v1"] } tempfile = "3" futures-util = "0.3.33" rmcp = { version = "3.0.1", default-features = false, features = ["server"] } +# Opt-in. Cargo rejects optional dev-dependencies, and an unconditional one +# unifies `tracedecay-query/search-eval` into every test target. Default and +# `production` builds leave it off, so the shipped CLI does not link it. +tracedecay-search-eval = { path = "../tracedecay-search-eval", version = "0.1.0", optional = true } # `kill(2)` for the daemon integration suites' physical-restart journeys # (tests/daemon_suite). Store-locality detection moved with the locator @@ -348,7 +360,6 @@ rmcp = { version = "3.0.1", default-features = false, features = ["server"] } libc = "0.2" [dev-dependencies] -tracedecay-search-eval = { path = "../tracedecay-search-eval", version = "0.1.0" } tree-sitter = "0.26" rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } diff --git a/crates/tracedecay/tests/product_surface_suite/main.rs b/crates/tracedecay/tests/product_surface_suite/main.rs index 9f0d827907..d95e6c0931 100644 --- a/crates/tracedecay/tests/product_surface_suite/main.rs +++ b/crates/tracedecay/tests/product_surface_suite/main.rs @@ -11,6 +11,9 @@ mod catalog_composition_contract; mod git_intelligence_regression; mod host_bundle_acceptance; mod native_integration_surface_mount; +// See `runtime_acceptance_suite`: the evaluator library is opt-in so transport +// tests do not compile the eval-only lexical projection. +#[cfg(feature = "search-eval")] mod packaged_search_evaluator; mod profile_backup_rehearsal_test; mod verified_profile_backup; diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/main.rs b/crates/tracedecay/tests/runtime_acceptance_suite/main.rs index d41e6cde75..7e9e00fe22 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/main.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/main.rs @@ -19,6 +19,10 @@ mod host_event_fixture_test; mod lifecycle_production_authority_test; mod private_route_restart_acceptance; mod runtime_surface_acceptance; +// Not implied by `test-transport`. An unconditional evaluator dependency +// unifies `tracedecay-query/search-eval` into every test target of this +// package, so the transport suites compile the eval-only lexical projection. +#[cfg(feature = "search-eval")] #[allow(clippy::option_env_unwrap)] mod search_eval_cli_test; #[cfg(unix)] From 690e84365d875954488298e7f07d835d67325dc2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:25:49 +0000 Subject: [PATCH 10/84] fix(retention): defer a census that races a missing path A generation-retention census that does not hold the store lock can observe a scope root the publisher has not created, or a file a peer unlinked after it was listed. That NotFound is the same deferral as a held writer, not a storage failure. The mounted journey waits until the superseded source is actually collectable. Co-authored-by: Zack Jackson --- .../src/code_index_generations.rs | 21 ++++++++-- .../code_index_generations/generation_scan.rs | 7 ++-- .../src/code_index_generations/locking.rs | 4 +- .../src/code_index_generations/tests.rs | 35 ++++++++++++++++ .../generation_retention_test.rs | 42 +++++++++++++++---- 5 files changed, 92 insertions(+), 17 deletions(-) diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations.rs b/crates/tracedecay-code-index-retention/src/code_index_generations.rs index b4976c6602..78c68296bd 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations.rs @@ -885,7 +885,7 @@ fn plan_code_generation_retention_with_verification_cancellable( Err(error) if error.kind() == std::io::ErrorKind::NotFound && active_pointer.is_none() => { None } - Err(error) => return Err(storage(error)), + Err(error) => return Err(deferred_if_absent(error)), }; let mut generations = BTreeMap::new(); let mut active_state_digest = None; @@ -1218,7 +1218,7 @@ fn sweep_unreferenced_generation_segments( continue; } let mut reader = CancellableGenerationManifestReaderV1 { - file: File::open(&path).map_err(storage)?, + file: File::open(&path).map_err(deferred_if_absent)?, hasher: Sha256::new(), is_cancelled, cancelled: false, @@ -1281,7 +1281,7 @@ fn sweep_unreferenced_generation_segments( if live_segments.contains(&format!("sha256:{digest}")) { continue; } - let metadata = path.symlink_metadata().map_err(storage)?; + let metadata = path.symlink_metadata().map_err(deferred_if_absent)?; if !metadata.file_type().is_file() { return Err(CodeGenerationRetentionErrorV1::UnsafeState(format!( "generation segment '{}' is not a regular file", @@ -1745,7 +1745,7 @@ fn read_active_pointer( store_root: &Path, ) -> Result { let path = store_root.join(ACTIVE_POINTER_FILE); - let bytes = std::fs::read(&path).map_err(storage)?; + let bytes = std::fs::read(&path).map_err(deferred_if_absent)?; serde_json::from_slice(&bytes).map_err(|error| { CodeGenerationRetentionErrorV1::UnsafeState(format!( "active pointer '{}' is corrupt: {error}", @@ -2033,5 +2033,18 @@ fn storage(error: impl std::fmt::Display) -> CodeGenerationRetentionErrorV1 { CodeGenerationRetentionErrorV1::Storage(error.to_string()) } +/// A path that is not there yet, or that a peer unlinked after this census +/// listed it, is not a broken disk. The publisher creates the scope root and +/// the sealed files under the store lock, then drops that lock; a census that +/// does not hold the lock can observe the gap. The next tick sees a stable +/// tree. Every other I/O failure stays a storage error. +pub(super) fn deferred_if_absent(error: std::io::Error) -> CodeGenerationRetentionErrorV1 { + if error.kind() == std::io::ErrorKind::NotFound { + CodeGenerationRetentionErrorV1::GenerationStoreBusy + } else { + storage(error) + } +} + #[cfg(test)] mod tests; diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/generation_scan.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/generation_scan.rs index 91b997173c..7c2923eaf2 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/generation_scan.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/generation_scan.rs @@ -7,7 +7,8 @@ use tracedecay_domain::canonical_text::{encode_tagged_lowercase_hex, is_lowercas use super::{ CodeGenerationRetentionErrorV1, GenerationDigestVerificationV1, - MAX_GENERATION_METADATA_PREFIX_BYTES, SealedGenerationManifestMetadataV1, storage, + MAX_GENERATION_METADATA_PREFIX_BYTES, SealedGenerationManifestMetadataV1, deferred_if_absent, + storage, }; const MAX_FORMAT_REVISION_PREFIX_BYTES: usize = 4 * 1024; @@ -16,7 +17,7 @@ pub(super) fn read_generation_format_revision( path: &Path, is_cancelled: &dyn Fn() -> bool, ) -> Result { - let mut file = File::open(path).map_err(storage)?; + let mut file = File::open(path).map_err(deferred_if_absent)?; let mut prefix = vec![0_u8; MAX_FORMAT_REVISION_PREFIX_BYTES]; let bytes_read = file.read(&mut prefix).map_err(storage)?; crate::hotpath_observe::retention_inspected(bytes_read as u64); @@ -40,7 +41,7 @@ pub(super) fn read_generation_metadata( is_cancelled: &dyn Fn() -> bool, ) -> Result<(u32, SealedGenerationManifestMetadataV1, String, u64), CodeGenerationRetentionErrorV1> { - let mut file = File::open(path).map_err(storage)?; + let mut file = File::open(path).map_err(deferred_if_absent)?; let size_bytes = file.metadata().map_err(storage)?.len(); let mut hasher = Sha256::new(); let mut prefix = Vec::with_capacity(MAX_GENERATION_METADATA_PREFIX_BYTES); diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/locking.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/locking.rs index 6bdc552abd..1d8d867a75 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/locking.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/locking.rs @@ -102,7 +102,7 @@ fn lock_file( } fn canonical_store_root(store_root: &Path) -> Result { - std::fs::canonicalize(store_root).map_err(storage) + std::fs::canonicalize(store_root).map_err(super::deferred_if_absent) } fn open_lock_file(path: &Path) -> Result { @@ -112,5 +112,5 @@ fn open_lock_file(path: &Path) -> Result { .write(true) .truncate(false) .open(path) - .map_err(storage) + .map_err(super::deferred_if_absent) } diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs index 2dbe91a0c7..116a8becc6 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs @@ -1501,6 +1501,41 @@ fn idle_maintenance_preparation_stays_metadata_only() { ); } +#[test] +fn preparation_defers_when_the_scope_root_does_not_exist_yet() { + let parent = tempfile::TempDir::new().expect("parent"); + let missing = parent.path().join("not-created"); + let error = prepare_next_code_generation_retention_cancellable( + &missing, + &BTreeSet::new(), + &|| false, + None, + ) + .expect_err("an unpublished scope root has no census"); + assert!( + matches!(error, CodeGenerationRetentionErrorV1::GenerationStoreBusy), + "a missing scope root is the publisher's create window, not a storage failure: {error:?}" + ); +} + +#[test] +fn preparation_defers_when_the_pointer_exists_without_its_generation_directory() { + let (store, _generations) = fixture_store(1); + std::fs::remove_dir_all(store.path().join(GENERATIONS_DIRECTORY)) + .expect("remove generation directory under a live pointer"); + let error = prepare_next_code_generation_retention_cancellable( + store.path(), + &BTreeSet::new(), + &|| false, + None, + ) + .expect_err("the generation directory is not durable yet"); + assert!( + matches!(error, CodeGenerationRetentionErrorV1::GenerationStoreBusy), + "a pointer without its generation directory is a torn publish, not a storage failure: {error:?}" + ); +} + #[test] fn metadata_only_segment_census_observes_at_most_one_directory_entry() { let store = tempfile::TempDir::new().expect("create unpublished store"); diff --git a/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs b/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs index b0d224dfec..0d8f8dd83d 100644 --- a/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs @@ -11,7 +11,8 @@ use super::journey_test_support::git; use super::*; use crate::daemon::maintenance::project_store_maintenance_lease; use tracedecay_code_index_retention::code_index_generations::{ - MAX_CODE_GENERATION_RETENTION_BATCH_V1, prepare_next_code_generation_retention_cancellable, + CodeGenerationRetentionErrorV1, MAX_CODE_GENERATION_RETENTION_BATCH_V1, + prepare_next_code_generation_retention_cancellable, }; use tracedecay_maintenance::tick::{MaintenanceContinuation, MaintenanceTickOutcome}; @@ -114,13 +115,38 @@ async fn mounted_code_generation_retention_continues_capped_segment_reclamation( &canonical_root, ); let graph_replay_pool_root = graph.db().database_path().with_extension("graph-replay"); - let plan = prepare_next_code_generation_retention_cancellable( - &code_store_root, - &BTreeSet::new(), - &|| false, - Some(&graph_replay_pool_root), - ) - .expect("code generation retention plan"); + // The serving id moves when the swap installs the generation. The sealed + // files and the replay pool are still being published and retired beside + // that swap, so one census can miss the scope root or a file it just + // listed. Those reads are `GenerationStoreBusy`, not a failed journey. + let plan = tokio::time::timeout(Duration::from_secs(20), async { + loop { + match prepare_next_code_generation_retention_cancellable( + &code_store_root, + &BTreeSet::new(), + &|| false, + Some(&graph_replay_pool_root), + ) { + Ok(plan) + if plan + .collectable_generations + .iter() + .any(|generation| generation.generation_id == first_source) => + { + return plan; + } + Ok(_) + | Err( + CodeGenerationRetentionErrorV1::GenerationStoreBusy + | CodeGenerationRetentionErrorV1::GraphReplayPoolBusy, + ) => {} + Err(error) => panic!("code generation retention plan: {error:?}"), + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("superseded source became collectable"); let first_candidate = plan .collectable_generations .iter() From fdb756d61634f545b68686e181e9ca6ef6ac658d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:26:55 +0000 Subject: [PATCH 11/84] fix(codex): keep prepare Ready when the plugin CLI is absent Codex prepare turned every resolution failure of the host CLI into DeferredUserAction, so Ready depended on PATH. Staging now returns Ready. Activation still reports HostCliUnavailable when codex cannot be resolved. Co-authored-by: Zack Jackson --- .../src/agents/codex.rs | 22 ++----- .../src/agents/codex/tests.rs | 57 +++++++++++-------- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/crates/tracedecay-agent-hosts/src/agents/codex.rs b/crates/tracedecay-agent-hosts/src/agents/codex.rs index 6141304f00..0599f13e06 100644 --- a/crates/tracedecay-agent-hosts/src/agents/codex.rs +++ b/crates/tracedecay-agent-hosts/src/agents/codex.rs @@ -88,24 +88,12 @@ impl AgentIntegration for CodexIntegration { &self, ctx: &InstallContext, ) -> Result { + // Staging is ready for Core apply to drive `codex plugin add`. Whether + // that binary resolves is activation's `HostCliUnavailable`, not a + // deferral: collapsing every resolution failure into + // `DeferredUserAction` made prepare's outcome a property of PATH and + // stopped the lifecycle before the host CLI could run. install_codex_plugin(&ctx.home, &ctx.tracedecay_bin)?; - // Core apply drives `codex plugin add` when the host CLI is present. - // When it is not, stop with the same backtick remediation preflight - // uses so operators (and lifecycle tests) can activate natively. - if plugin_registry::require_codex_plugin_cli().is_err() { - let marketplace_name = codex_exact_personal_marketplace_name(&ctx.home) - .ok() - .flatten() - .unwrap_or_else(|| codex_cached_marketplace_name(&ctx.home)); - return Ok(NonInteractiveInstallOutcome::DeferredUserAction( - DeferredUserAction { - remediation: format!( - "Codex activates plugins through its native cache. Run `codex plugin add tracedecay@{marketplace_name}` after TraceDecay stages the source package." - ), - staged_paths: Vec::new(), - }, - )); - } Ok(NonInteractiveInstallOutcome::Ready) } diff --git a/crates/tracedecay-agent-hosts/src/agents/codex/tests.rs b/crates/tracedecay-agent-hosts/src/agents/codex/tests.rs index 1a6f1b1c9a..7d2d42ffbc 100644 --- a/crates/tracedecay-agent-hosts/src/agents/codex/tests.rs +++ b/crates/tracedecay-agent-hosts/src/agents/codex/tests.rs @@ -979,34 +979,23 @@ fn codex_preflight_reports_inactive_cache_without_interactive_guidance() { assert!(CodexIntegration.interactive_removal_guidance().is_none()); } -/// Install an executable `codex` on the host-program search path only. +/// Restrict host-program resolution to an empty directory. /// -/// Preparation is `Ready` exactly when Codex's own plugin CLI is present, so -/// the outcome under test is a property of the environment, not of the host -/// integration. CI runners carry no `codex` binary while a developer box -/// usually does; pin it here instead of reading whichever the machine has. -/// Only host program resolution sees this directory, the process `PATH` is -/// untouched. -fn install_fake_codex_cli( - dir: &Path, -) -> tracedecay_runtime_core::config::HostProgramSearchPathGuard { - let binary = dir.join(format!("codex{}", std::env::consts::EXE_SUFFIX)); - std::fs::write(&binary, "#!/bin/sh\nexit 0\n").unwrap(); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let mut permissions = std::fs::metadata(&binary).unwrap().permissions(); - permissions.set_mode(0o755); - std::fs::set_permissions(&binary, permissions).unwrap(); - } - tracedecay_runtime_core::config::HostProgramSearchPathGuard::set(dir) +/// The directory has to outlive the guard. An ambient `codex` on the process +/// PATH must not be able to change the outcome under test. +fn hide_host_programs() -> ( + tempfile::TempDir, + tracedecay_runtime_core::config::HostProgramSearchPathGuard, +) { + let dir = tempfile::tempdir().unwrap(); + let guard = tracedecay_runtime_core::config::HostProgramSearchPathGuard::set(dir.path()); + (dir, guard) } #[test] fn prepare_stages_the_source_and_returns_ready_for_cli_activation() { let home = tempfile::tempdir().unwrap(); - let cli_dir = tempfile::tempdir().unwrap(); - let _codex_cli = install_fake_codex_cli(cli_dir.path()); + let (_empty_path, _host_programs) = hide_host_programs(); // Pre-existing user config: preparation runs before the component // transaction stages `config.toml`, so it must not write there, hook // trust is recorded by activation, inside the rollback boundary. @@ -1017,7 +1006,10 @@ fn prepare_stages_the_source_and_returns_ready_for_cli_activation() { let outcome = CodexIntegration .prepare_non_interactive_install(&install_ctx(home.path())) .unwrap(); - assert!(matches!(outcome, NonInteractiveInstallOutcome::Ready)); + assert!( + matches!(outcome, NonInteractiveInstallOutcome::Ready), + "staging Codex must be Ready even when no host CLI resolves, got {outcome:?}" + ); assert!(codex_plugin_manifest_path(home.path()).is_file()); assert!(codex_personal_marketplace_path(home.path()).is_file()); assert_eq!( @@ -1027,6 +1019,25 @@ fn prepare_stages_the_source_and_returns_ready_for_cli_activation() { ); } +/// A missing plugin CLI is an unavailable host, not a successful deferral. +/// Activation is the boundary that drives `codex plugin add`. +#[test] +fn activation_names_a_missing_plugin_cli_instead_of_deferring() { + let home = tempfile::tempdir().unwrap(); + let (_empty_path, _host_programs) = hide_host_programs(); + + let error = CodexIntegration + .activate_deployed_host_registration(&install_ctx(home.path())) + .expect_err("activation without a Codex CLI must fail"); + let TraceDecayError::HostCliUnavailable { program, lifecycle } = error else { + panic!( + "a missing Codex plugin CLI must stay HostCliUnavailable, not another error: {error}" + ); + }; + assert_eq!(program, "codex"); + assert_eq!(lifecycle, "codex plugin lifecycle"); +} + /// Activation must record hook trust even when Codex already reports the /// plugin natively active (no `codex plugin add` run): an already-current /// install can still carry missing or stale trust, and the canonical From df9f190eeaad533d00b65b19fbc0d4f47426136b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:29:34 +0000 Subject: [PATCH 12/84] fix(extraction): stop bare receivers inventing callers Dotted Rust calls emitted the method's simple name, so a unique same-file callable of that name became a caller. Bind self through the enclosing impl or trait, and keep Type::method for stated receivers. Co-authored-by: Zack Jackson --- .../src/rust_extractor.rs | 90 ++++++++++----- .../tests/main/rust.rs | 105 +++++++++++++++++- crates/tracedecay-code-index/src/chunks.rs | 96 +++++++++++++++- 3 files changed, 256 insertions(+), 35 deletions(-) diff --git a/crates/tracedecay-code-extraction/src/rust_extractor.rs b/crates/tracedecay-code-extraction/src/rust_extractor.rs index dc6537912e..17261a1a99 100644 --- a/crates/tracedecay-code-extraction/src/rust_extractor.rs +++ b/crates/tracedecay-code-extraction/src/rust_extractor.rs @@ -28,8 +28,8 @@ struct ShadowedCallNames { } /// Receiver bindings whose type the function body states outright: typed -/// parameters, typed `let`s, and `let`s initialised by a struct literal -/// (`T { .. }`, possibly behind `?`). A dotted +/// parameters, typed `let`s, `let`s initialised by a struct literal +/// (`T { .. }`, possibly behind `?`), and `self` in a method. A dotted /// call on such a binding also names the method by its type /// (`builder.build()` → `ignore::WalkBuilder::build`), which is the only form /// the resolver can bind across files. Method calls and constructor-like names @@ -1579,24 +1579,11 @@ impl RustExtractor { column: child.start_position().column as u32, file_path: state.file_path.clone(), }); - // For dot-calls (e.g. `instance.method()`), also emit - // a ref with just the method name so the resolver can - // match it against impl method definitions. - if let Some(method_name) = callee_name.rsplit('.').next() - && method_name != callee_name - { - state.unresolved_refs.push(UnresolvedRef { - from_node_id: fn_node_id.to_string(), - reference_name: method_name.to_string(), - reference_kind: EdgeKind::Calls, - line: child.start_position().row as u32, - column: child.start_position().column as u32, - file_path: state.file_path.clone(), - }); - } - // A dotted call on a binding with a stated type also - // names the method through its type, the only form - // that binds across files. + // The simple name of a dotted call is not itself a call. + // `items.push()` must not bind a same-file `fn push`. + // Only a stated receiver type names the method + // (`Rows::len`), which is also the form that binds + // across files. if let Some(typed_method) = Self::typed_receiver_method(state, callee, receivers) { @@ -1664,13 +1651,51 @@ impl RustExtractor { } let value = callee.child_by_field_name("value")?; let field = callee.child_by_field_name("field")?; - if value.kind() != "identifier" || field.kind() != "field_identifier" { + if field.kind() != "field_identifier" { return None; } - let type_path = receivers.type_of(state.node_text(value))?; + // `self` is its own token, not an identifier. Both name a binding. + let receiver_name = match value.kind() { + "identifier" | "self" => state.node_text(value), + _ => return None, + }; + let type_path = receivers.type_of(receiver_name)?; Some(format!("{type_path}::{}", state.node_text(field))) } + /// The type `self` names in the enclosing impl or trait. + /// + /// Trait impls store `` so the method keeps a UFCS name. + /// `self` still names `Type`, the path a call site writes and the alias + /// same-file resolution binds. + fn enclosing_receiver_type(state: &ExtractionState<'_>) -> Option { + let (name, id) = state + .node_stack + .iter() + .rev() + .find(|(_, id)| id.starts_with("impl:") || id.starts_with("trait:"))?; + let type_name = if id.starts_with("impl:") { + match name + .strip_prefix('<') + .and_then(|inner| inner.split_once(" as ")) + { + Some((type_name, _)) => type_name.trim(), + None => name.as_str(), + } + } else { + name.as_str() + }; + if type_name.is_empty() + || type_name == "Self" + || type_name == "" + || type_name == "" + { + None + } else { + Some(type_name.to_owned()) + } + } + /// Records every binding the function introduces with the type it states, /// or `None` for a binding whose type the syntax does not state (pattern /// destructuring, `if let`, `match` arms, closure parameters, `for`). @@ -1681,6 +1706,11 @@ impl RustExtractor { receivers: &mut ReceiverTypes, ) { match node.kind() { + "self_parameter" => { + if let Some(type_path) = Self::enclosing_receiver_type(state) { + receivers.record("self".to_owned(), Some(type_path)); + } + } "parameter" => { if let Some(pattern) = node.child_by_field_name("pattern") { let type_path = node @@ -1737,15 +1767,15 @@ impl RustExtractor { } } - /// A bare identifier pattern takes `type_path`; every identifier inside any - /// other pattern is bound with an unknown type. + /// A bare identifier or `self` pattern takes `type_path`; every identifier + /// inside any other pattern is bound with an unknown type. fn record_receiver_pattern( state: &ExtractionState<'_>, pattern: TsNode<'_>, type_path: Option, receivers: &mut ReceiverTypes, ) { - if pattern.kind() == "identifier" { + if pattern.kind() == "identifier" || pattern.kind() == "self" { receivers.record(state.node_text(pattern).to_owned(), type_path); return; } @@ -1763,10 +1793,18 @@ impl RustExtractor { /// The nominal type path a type annotation names, seen through references, /// generic arguments, and `dyn`/`impl` trait objects; `None` for tuples, /// slices, function pointers, and anything else without one nominal head. + /// `Self` is the enclosing impl or trait type when one is on the stack. fn stated_type_path(state: &ExtractionState<'_>, ty: TsNode<'_>) -> Option { match ty.kind() { "type_identifier" | "scoped_type_identifier" => { - Some(state.node_text(ty).to_owned()).filter(|path| path != "Self") + let path = state.node_text(ty); + if path == "Self" { + // `Self` in an annotation is the enclosing impl or trait, + // not a type the file declared under that name. + Self::enclosing_receiver_type(state) + } else { + Some(path.to_owned()) + } } "reference_type" | "generic_type" => ty .child_by_field_name("type") diff --git a/crates/tracedecay-code-extraction/tests/main/rust.rs b/crates/tracedecay-code-extraction/tests/main/rust.rs index 9a3a5f7288..7e74529f99 100644 --- a/crates/tracedecay-code-extraction/tests/main/rust.rs +++ b/crates/tracedecay-code-extraction/tests/main/rust.rs @@ -1131,10 +1131,109 @@ fn use_foo() { ref_names.contains(&"Foo::new"), "expected Foo::new call, got: {ref_names:?}" ); - // f.bar() should also produce "bar" (method-name hint). assert!( - ref_names.contains(&"bar"), - "expected 'bar' method-name ref from f.bar(), got: {ref_names:?}" + ref_names.contains(&"f.bar"), + "the receiver-dotted form remains: {ref_names:?}" + ); + // `Foo::new()` does not state that `f` is Foo, and `bar` is the method of + // an impl in this file. Emitting the simple name would invent that caller. + assert!( + !ref_names.contains(&"bar"), + "untyped f.bar() must not emit a bare method name: {ref_names:?}" + ); + assert!( + !ref_names.contains(&"Foo::bar"), + "constructor-like Foo::new() must not fabricate a Foo receiver: {ref_names:?}" + ); +} + +#[test] +fn bare_receiver_calls_name_self_without_the_method_simple_name() { + let source = r#" +fn prepare(value: i32) {} +fn push(value: i32) {} + +struct Rows; +impl Rows { + fn len(&self) -> usize { 0 } + fn measure(&self) -> usize { self.len() } + fn via_explicit(self: &Self) -> usize { self.len() } +} +trait Span {} +impl Span for Rows { + fn wide(&self) -> usize { self.len() } +} + +fn caller(items: Vec, rows: Rows) { + let foreign = make(); + foreign.prepare(1); + items.push(1); + prepare(1); + push(1); + rows.len(); +} +fn make() -> Vec { Vec::new() } +"#; + let result = RustExtractor.extract("src/lib.rs", source); + assert!(result.errors.is_empty(), "{:?}", result.errors); + + let from = |name: &str| { + let function = result + .nodes + .iter() + .find(|node| { + matches!(node.kind, NodeKind::Function | NodeKind::Method) && node.name == name + }) + .unwrap_or_else(|| panic!("{name} is extracted")); + result + .unresolved_refs + .iter() + .filter(|reference| { + reference.reference_kind == EdgeKind::Calls && reference.from_node_id == function.id + }) + .map(|reference| reference.reference_name.as_str()) + .collect::>() + }; + + let measure = from("measure"); + assert!( + measure.contains(&"self.len") && measure.contains(&"Rows::len"), + "{measure:?}" + ); + assert!( + !measure.contains(&"len"), + "self.len() must not emit the bare method name: {measure:?}" + ); + + let via_explicit = from("via_explicit"); + assert!( + via_explicit.contains(&"Rows::len"), + "self: &Self still names the enclosing type: {via_explicit:?}" + ); + + let wide = from("wide"); + assert!( + wide.contains(&"Rows::len"), + "self inside `impl Span for Rows` names Rows, not Span: {wide:?}" + ); + assert!(!wide.contains(&"Span::len"), "{wide:?}"); + + let caller = from("caller"); + assert!(caller.contains(&"prepare"), "{caller:?}"); + assert!(caller.contains(&"push"), "{caller:?}"); + assert!(caller.contains(&"Rows::len"), "{caller:?}"); + assert!(caller.contains(&"Vec::push"), "{caller:?}"); + assert!(caller.contains(&"foreign.prepare"), "{caller:?}"); + assert!(caller.contains(&"items.push"), "{caller:?}"); + assert_eq!( + caller.iter().filter(|name| **name == "prepare").count(), + 1, + "foreign.prepare() invented a second prepare call: {caller:?}" + ); + assert_eq!( + caller.iter().filter(|name| **name == "push").count(), + 1, + "items.push() invented a second push call: {caller:?}" ); } diff --git a/crates/tracedecay-code-index/src/chunks.rs b/crates/tracedecay-code-index/src/chunks.rs index 8cdc6956be..85f27ed90c 100644 --- a/crates/tracedecay-code-index/src/chunks.rs +++ b/crates/tracedecay-code-index/src/chunks.rs @@ -2276,12 +2276,11 @@ fn resolve_file_references( .and_modify(|entry| *entry = None) .or_insert(Some(symbol)); } - // Extractors emit a bare method-name duplicate alongside every dotted - // receiver call (`self.rows.push(row)` → `self.rows.push` + `push`) so an - // in-file method definition can still match. Index those duplicates by - // their call site: a duplicate that binds back to its own enclosing symbol - // is a receiver whose type is unknown (usually a container or another - // struct's method sharing the name), not evidence of recursion. + // A dotted call used to also emit its method's simple name + // (`self.rows.push(row)` → `self.rows.push` + `push`) so an in-file method + // could match. That duplicate binds the enclosing symbol when the names + // coincide, which is a receiver call, not recursion. Rust extraction no + // longer emits it; keep the skip so a duplicate cannot invent one. let dotted_duplicate_sites = unresolved .iter() .filter(|reference| reference.reference_name.contains('.')) @@ -4368,6 +4367,91 @@ pub fn real_symbol() {} assert_ne!(calls[0].evidence_span, calls[1].evidence_span); } + #[test] + fn bare_receiver_method_call_does_not_invent_a_same_file_caller() { + let source = concat!( + "fn prepare(value: i32) {}\n", + "fn push(value: i32) {}\n", + "\n", + "struct Rows;\n", + "impl Rows {\n", + " fn len(&self) -> usize { 0 }\n", + " fn measure(&self) -> usize { self.len() }\n", + "}\n", + "trait Span {}\n", + "impl Span for Rows {\n", + " fn wide(&self) -> usize { self.len() }\n", + "}\n", + "\n", + "fn caller(items: Vec, rows: Rows) {\n", + " let foreign = make();\n", + " foreign.prepare(1);\n", + " items.push(1);\n", + " prepare(1);\n", + " push(1);\n", + " rows.len();\n", + "}\n", + "fn make() -> Vec { Vec::new() }\n", + ); + let file = validated_file("src/lib.rs", source.as_bytes()); + let batch = batch_for(&file, ParseOutcomeV1::Complete); + let artifacts = chunker() + .index_file(&file, &batch, &rust_descriptor(), &NeverCancelled) + .expect("indexing succeeds"); + let qualified = |occurrence: &SymbolOccurrenceId| { + artifacts + .symbols + .iter() + .find(|symbol| &symbol.occurrence == occurrence) + .map(|symbol| symbol.qualified_name.as_str()) + .unwrap_or("") + }; + let mut calls = artifacts + .edges + .iter() + .filter(|edge| edge.kind == RelationEdgeKindV1::Calls) + .map(|edge| { + ( + qualified(&edge.from_occurrence).to_owned(), + qualified(&edge.to_occurrence).to_owned(), + ) + }) + .collect::>(); + calls.sort(); + + assert_eq!( + calls, + vec![ + ( + "src/lib.rs::::wide".to_owned(), + "src/lib.rs::Rows::len".to_owned(), + ), + ( + "src/lib.rs::Rows::measure".to_owned(), + "src/lib.rs::Rows::len".to_owned(), + ), + ( + "src/lib.rs::caller".to_owned(), + "src/lib.rs::Rows::len".to_owned(), + ), + ( + "src/lib.rs::caller".to_owned(), + "src/lib.rs::make".to_owned(), + ), + ( + "src/lib.rs::caller".to_owned(), + "src/lib.rs::prepare".to_owned(), + ), + ( + "src/lib.rs::caller".to_owned(), + "src/lib.rs::push".to_owned() + ), + ], + "a bare receiver must not add a same-file caller; self and typed \ + bindings still bind: {calls:?}" + ); + } + #[test] fn rust_type_path_alias_parses_ufcs_trait_impl_methods() { assert_eq!( From 391c6a4dddd027036dbf7c5d060c4b3bcc893009 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 06:29:52 +0000 Subject: [PATCH 13/84] test(mcp): prove remote status across the tools/call boundary `tracedecay_remote_status` was proved only below the transport. The handler unit test and `remote_status_dispatch_tests` call the dispatch function directly, and `daemon::remote_protocol_tests` reads the session runtime registry. Nothing exercised the tool through a real `tools/call`, so the two typed no-plane outcomes could diverge at the boundary without a failing test. Add the transport-level cases the existing suites do not cover: - A production composition harness mounts the Remote Brain reader with no listener and no registered node, so `tools/call` must render `unconfigured` in both markdown and JSON. - A direct protocol server never installs the reader, so the same call over JSON-RPC must render `unavailable`, with no `isError`. Both cases also assert the response carries exactly one content block, pinning that a typed read attaches no banner or token footer. The configured (`observed`) plane is left to the two suites that can mount one cheaply: the harness exposes no registry accessor, so provisioning a node here would duplicate `remote_protocol_tests` at the cost of reproducing daemon authority acquisition, a signed grant, and enrollment admission in an integration fixture. Co-Authored-By: Claude Fable 5.1 --- .../tests/mcp_suite/mcp_handler_test.rs | 1 + .../mcp_handler_test/remote_status_test.rs | 164 ++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 crates/tracedecay/tests/mcp_suite/mcp_handler_test/remote_status_test.rs diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test.rs index ec5f21ffcf..34ae994892 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test.rs @@ -59,6 +59,7 @@ mod project_context_test; mod project_list_test; #[cfg(feature = "test-transport")] mod project_search_behavior_test; +mod remote_status_test; #[cfg(feature = "test-transport")] mod rename_symbol_test; mod retrieve_truncation_test; diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/remote_status_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/remote_status_test.rs new file mode 100644 index 0000000000..078dd96375 --- /dev/null +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/remote_status_test.rs @@ -0,0 +1,164 @@ +//! Real `tools/call` coverage for `tracedecay_remote_status`. +//! +//! The production daemon mounts the Remote Brain reader. With no listener and +//! no registered node, that reader is `unconfigured`. A direct server never +//! installs the reader, so the same call is `unavailable`. Neither outcome is +//! an empty success or a semantic tool error. +//! +//! The configured (`observed`) plane is proved at the two seams that can mount +//! one cheaply: `daemon::remote_protocol_tests` provisions a real node and +//! serving listener against the session runtime registry, and +//! `mcp::tools::handlers::info::remote_status_dispatch_tests` drives an +//! observed reader through dispatch. This suite owns the transport boundary +//! those two do not cross. + +use std::path::PathBuf; +use std::process::Command; + +use serde_json::{Value, json}; +use tracedecay::daemon::ProductionProjectCompositionHarnessV1; + +use crate::common; +use crate::fixture; +use crate::mcp_server_test::support::{ + jsonrpc_request, response_with_id, run_server_with_messages, setup_server, successful_tool_text, +}; +use crate::support::{TestTempDir, test_temp_dir}; + +const UNCONFIGURED_JSON: &str = r#"{"kind":"unconfigured"}"#; +const UNAVAILABLE_JSON: &str = r#"{"kind":"unavailable"}"#; +const UNCONFIGURED_MARKDOWN: &str = "**kind:** unconfigured\n"; +const UNAVAILABLE_MARKDOWN: &str = "**kind:** unavailable\n"; + +struct MountedDaemon { + harness: ProductionProjectCompositionHarnessV1, + project: PathBuf, + _isolation: TestTempDir, +} + +async fn mount_daemon_without_remote_plane() -> MountedDaemon { + let isolation = test_temp_dir(); + let project = isolation.path().join("project"); + std::fs::create_dir_all(&project).expect("remote-status project directory"); + fixture::write_indexed_fixture_sources(&project); + for args in [ + vec!["init", "-q"], + vec!["add", "."], + vec![ + "-c", + "user.name=TraceDecay Test", + "-c", + "user.email=tracedecay@example.invalid", + "commit", + "-qm", + "remote status fixture", + ], + ] { + let status = Command::new(common::git_program()) + .args(args) + .current_dir(&project) + .status() + .expect("git"); + assert!(status.success(), "git must succeed for {project:?}"); + } + let harness = Box::pin( + ProductionProjectCompositionHarnessV1::open_for_session_retrieval( + isolation.path(), + [project.clone()], + ), + ) + .await + .expect("production composition"); + MountedDaemon { + harness, + project, + _isolation: isolation, + } +} + +fn status_text(result: &Value) -> &str { + let content = result["content"] + .as_array() + .unwrap_or_else(|| panic!("remote status returned no content array: {result}")); + assert_eq!( + content.len(), + 1, + "remote status must not attach banners or token footers: {result}" + ); + assert!( + result.get("isError").is_none(), + "a typed remote-status read is not a semantic tool error: {result}" + ); + content[0]["text"] + .as_str() + .unwrap_or_else(|| panic!("remote status text content missing: {result}")) +} + +async fn daemon_status(mounted: &MountedDaemon, arguments: Value) -> Value { + let response = mounted + .harness + .call_tool(&mounted.project, "tracedecay_remote_status", arguments) + .await + .expect("production tools/call"); + assert!( + response.error.is_none(), + "production remote status must succeed: {:?}", + response.error + ); + response + .result + .unwrap_or_else(|| panic!("production remote status missing result")) +} + +#[tokio::test] +async fn production_daemon_reports_unconfigured_remote_plane() { + let mounted = mount_daemon_without_remote_plane().await; + + let markdown = daemon_status(&mounted, json!({})).await; + assert_eq!(status_text(&markdown), UNCONFIGURED_MARKDOWN); + + let json_result = daemon_status(&mounted, json!({"format": "json"})).await; + assert_eq!(status_text(&json_result), UNCONFIGURED_JSON); +} + +#[tokio::test] +async fn direct_server_reports_unmounted_remote_authority() { + let (server, _dir) = setup_server().await; + let responses = run_server_with_messages( + server, + vec![ + jsonrpc_request( + json!(1), + "tools/call", + json!({ + "name": "tracedecay_remote_status", + "arguments": {} + }), + ), + jsonrpc_request( + json!(2), + "tools/call", + json!({ + "name": "tracedecay_remote_status", + "arguments": {"format": "json"} + }), + ), + ], + ) + .await; + + let markdown = response_with_id(&responses, json!(1)); + assert_eq!( + successful_tool_text(&markdown, "markdown remote status"), + UNAVAILABLE_MARKDOWN + ); + let json_response = response_with_id(&responses, json!(2)); + assert_eq!( + successful_tool_text(&json_response, "json remote status"), + UNAVAILABLE_JSON + ); + assert!( + json_response["result"].get("isError").is_none(), + "an unmounted remote authority is a typed read, not a tool error: {json_response}" + ); +} From e89842192367178d665ca409b40b3a5a28f3f6e2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:30:28 +0000 Subject: [PATCH 14/84] fix(code-index): keep sealed generation when proof expires A seal or clone backfill outlives the 30s freshness window. Expiry and a predecessor witness used to clear the newer generation and reseal it. Unchanged sealed bytes now rebind that proof instead. Co-authored-by: Zack Jackson --- .../src/code_index_scheduler/reconcile.rs | 183 +++++++++++++++--- .../code_index_scheduler/registry/mount.rs | 29 +-- .../code_index_scheduler/tests/reconcile.rs | 133 +++++++++++++ 3 files changed, 299 insertions(+), 46 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index c2c7eecdde..878c9b53e9 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -608,6 +608,37 @@ impl SourceFreshnessFenceV1 { }) && self.snapshot_is_recently_verified(&state, project_root, shutting_down) } + + /// Whether the last completed proof was sealed from exactly this snapshot. + /// + /// Clock age is not part of the answer. A seal or clone backfill can + /// outlive the admission window without the snapshot changing identity. + pub(super) fn proof_describes_snapshot( + &self, + snapshot_content_identity: &ContentDigest, + ) -> bool { + let state = self.snapshot(); + state.verified_against_source + && state.source_witness.as_ref().is_some_and(|witness| { + witness + .content_manifest + .describes_snapshot(snapshot_content_identity) + }) + } + + /// Refresh the admission clock and the git-metadata sample after the + /// sealed digests still matched. The content witness and reconciled + /// epoch stay put: this is the same proof, not a new generation. + fn rebind_admission_clock(&self, git_metadata: identity::GitMetadataFingerprintV1) { + let micros = now_micros().0; + let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + state.git_metadata = git_metadata; + state.last_reconciled_at = Instant::now(); + state.verified_against_source = true; + state.freshness_unknown = false; + self.last_reconciled_at_micros + .store(micros, Ordering::Release); + } } /// What the cheap Git/stat freshness ladder concluded about the retained @@ -1737,6 +1768,45 @@ impl CodeIndexWorktreeSchedulerV1 { Ok(Some(outcome)) } + /// Record that `metadata` is the generation the live worktree still seals. + /// + /// The in-memory fence takes this snapshot. The disk witness, when one + /// exists, is rewritten to this generation id so the next open does not + /// treat the predecessor's proof as a reason to drop it and reseal. + fn accept_unchanged_sealed_snapshot( + &mut self, + metadata: &VerifiedSealedTextGenerationMetadataV1, + git_metadata: identity::GitMetadataFingerprintV1, + stat_signature: String, + source_manifest: SourceContentManifestV1, + prior_witness: Option<&RestoreFreshnessWitnessV1>, + ) -> CodeIndexReconcileOutcomeV1 { + let snapshot_content_identity = metadata.snapshot().content_identity.clone(); + self.latest_content_identity = Some(snapshot_content_identity.clone()); + self.mark_reconciled_retained_generation_state( + git_metadata.clone(), + Some(ReconciledSourceWitnessV1 { + stat_signature: stat_signature.clone(), + content_manifest: source_manifest, + }), + ); + if let Some(prior) = prior_witness { + RestoreFreshnessWitnessV1 { + generation_id: metadata.manifest().generation_id.as_str().to_owned(), + git_metadata_signature: git_metadata.stable_signature(), + stat_signature, + repository_parse_identity_digest: prior.repository_parse_identity_digest.clone(), + ignored_source_admissions_digest: prior.ignored_source_admissions_digest.clone(), + ignored_source_paths: Vec::new(), + } + .persist(&self.store_root); + } + CodeIndexReconcileOutcomeV1::Noop(CodeIndexNoopEvidenceV1 { + snapshot_content_identity, + overflow_reconciled: false, + }) + } + pub(super) fn reconcile_retained_text_generation_with( &mut self, metadata: &VerifiedSealedTextGenerationMetadataV1, @@ -1758,10 +1828,14 @@ impl CodeIndexWorktreeSchedulerV1 { .observe_retained_text_compatibility(metadata) .is_reusable(); let witness = RestoreFreshnessWitnessV1::load(&self.store_root); - if witness.as_ref().is_some_and(|witness| { - witness.generation_id != metadata.manifest().generation_id.as_str() - || !witness.ignored_source_paths.is_empty() - }) || !self.ignored_source_admissions.is_empty() + // A predecessor freshness witness is not a reason to drop this + // generation. It names the proof that sealed an earlier snapshot. + // Ignored-source rosters still require the complete capture: their + // digest is not the ordinary file manifest this path compares. + if witness + .as_ref() + .is_some_and(|witness| !witness.ignored_source_paths.is_empty()) + || !self.ignored_source_admissions.is_empty() { return Ok(None); } @@ -1788,37 +1862,36 @@ impl CodeIndexWorktreeSchedulerV1 { // generation's sealed file digests; its matching stat signature is // the negative cache that lets a moved tree skip the byte comparison. let source_manifest = SourceContentManifestV1::for_snapshot(metadata.snapshot()); - if retained_is_reusable + let sealed_bytes_match = retained_is_reusable && !has_hints - && let Some(witness) = witness.as_ref() - && witness.git_metadata_signature == sampled_metadata.stable_signature() - && witness.stat_signature == sampled_sweep.signature && sampled_sweep.content_matches( &self.project_root, &source_manifest, &self.shutting_down, - ) - { - let snapshot_content_identity = metadata.snapshot().content_identity.clone(); - self.latest_content_identity = Some(snapshot_content_identity.clone()); - self.mark_reconciled_retained_generation_state( - sampled_metadata, - Some(ReconciledSourceWitnessV1 { - stat_signature: sampled_sweep.signature, - content_manifest: source_manifest, - }), ); - return Ok(Some(CodeIndexReconcileOutcomeV1::Noop( - CodeIndexNoopEvidenceV1 { - snapshot_content_identity, - overflow_reconciled: false, - }, + let quiet_witness = sealed_bytes_match + && witness.as_ref().is_some_and(|witness| { + witness.git_metadata_signature == sampled_metadata.stable_signature() + && witness.stat_signature == sampled_sweep.signature + }); + // Graph-on refuses to decode the sealed generation just because the + // predecessor witness, or a git-index mtime this seal itself moved, + // does not name this generation. The sealed digests are the proof. + // Graph-off still captures so a metadata-only drift is verified + // without a full decode when the quiet witness is absent. + if sealed_bytes_match && (quiet_witness || !rebuild_changed_source_without_decode) { + return Ok(Some(self.accept_unchanged_sealed_snapshot( + metadata, + sampled_metadata, + sampled_sweep.signature, + source_manifest, + witness.as_ref(), ))); } - // A compatible generation whose witness did not prove a quiet tree - // falls through to the full graph-on reconcile. An incompatible - // lightweight owner rebuilds here without decoding the retained graph. + // A compatible generation whose bytes moved falls through to the full + // graph-on reconcile. An incompatible lightweight owner rebuilds here + // without decoding the retained graph. if retained_is_reusable && !rebuild_changed_source_without_decode { return Ok(None); } @@ -2813,6 +2886,51 @@ impl CodeIndexWorktreeSchedulerV1 { .source_currency_witness_for(generation_id, snapshot_content_identity) } + /// Bind a sealed snapshot to the source proof, renewing an expired clock + /// when the sealed digests still match. + /// + /// The admission window is 30s. A graph seal and the clone-fingerprint + /// backfill both outlive it under load. Treating that expiry as "this + /// generation is not the proof" cleared the serving witness and the next + /// pass resealed the same snapshot. A hook epoch or a digest mismatch + /// still refuses; only an unchanged sealed snapshot keeps its generation. + pub(super) fn currency_witness_for_sealed_snapshot( + &self, + generation_id: &CodeGenerationId, + snapshot_content_identity: &ContentDigest, + ) -> Option { + if self.shutting_down.load(Ordering::Acquire) { + return None; + } + if self.freshness_fence.serves_recently_verified_source( + snapshot_content_identity, + &self.project_root, + &self.shutting_down, + ) { + return self + .freshness_fence + .source_currency_witness_for(generation_id, snapshot_content_identity); + } + if !self + .freshness_fence + .proof_describes_snapshot(snapshot_content_identity) + || self.freshness_fence.source_change_pending() + { + return None; + } + let freshness = self.freshness_fence.snapshot(); + if !self.source_witness_matches_worktree(&freshness) { + return None; + } + // Sample after the walk. `gix::open` inside the digest comparison can + // move index metadata; storing the post-walk sample is what keeps the + // next probe from calling that side effect a new generation. + let git_metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); + self.freshness_fence.rebind_admission_clock(git_metadata); + self.freshness_fence + .source_currency_witness_for(generation_id, snapshot_content_identity) + } + /// A cheap stat-level (path, mtime, size) signature of the present source /// candidates. It opens gix and runs stat-based status (no byte reads, no /// content hashing). A changed signature skips straight to reconcile; an @@ -3252,6 +3370,19 @@ impl CodeIndexWorktreeSchedulerV1 { self.publication.sealed_decode_count() } + /// Age the admission clock past its own threshold without touching source. + #[cfg(test)] + pub(super) fn expire_source_proof_for_test(&self) { + let mut state = self + .freshness_fence + .state + .lock() + .unwrap_or_else(PoisonError::into_inner); + state.last_reconciled_at = Instant::now() + .checked_sub(state.staleness_threshold + Duration::from_secs(1)) + .unwrap_or_else(Instant::now); + } + #[cfg(any(test, feature = "test-helpers"))] pub fn poison_decoded_publication_cache_for_test(&self) { self.publication.poison_decoded_cache_for_test(); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs index ac239ce49c..0a726b2573 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs @@ -1801,8 +1801,6 @@ impl CodeIndexSchedulerRegistryV1 { let text_generation = Arc::clone(&worker_text_generation); let serving_seats = Arc::clone(&worker_serving_seats); let serving_generation_changed = worker_serving_generation_changed.clone(); - let source_freshness = worker_source_freshness.clone(); - let project_root = worker_project_root.clone(); let text_latest = latest.clone(); let latest = latest.clone(); let shutting_down = Arc::clone(&worker_shutting_down); @@ -1836,13 +1834,14 @@ impl CodeIndexSchedulerRegistryV1 { // proofs to the seat. Asking the fence whether it // has verified *this* sealed snapshot is what makes // the binding truthful for a seat this pass did not - // publish. - let pass_proves_latest = source_freshness - .serves_recently_verified_source( - &latest.generation().snapshot().content_identity, - &project_root, - &shutting_down, - ); + // publish. An expired clock, or a git-index sample + // this seal moved, is not a different snapshot: + // dropping the witness here is how a newer + // generation stayed unserved through clone backfill. + let sealed_currency = scheduler.currency_witness_for_sealed_snapshot( + &latest.generation().manifest().generation_id, + &latest.generation().snapshot().content_identity, + ); let mut serving = serving_generation .write() .unwrap_or_else(std::sync::PoisonError::into_inner); @@ -1883,17 +1882,7 @@ impl CodeIndexSchedulerRegistryV1 { *serving_source_witness .write() .unwrap_or_else(std::sync::PoisonError::into_inner) = - pass_proves_latest - .then(|| { - source_freshness.source_currency_witness_for( - &latest.generation().manifest().generation_id, - &latest - .generation() - .snapshot() - .content_identity, - ) - }) - .flatten(); + sealed_currency; } // The durable pointer names a successor, so no // proof of this seat's currency exists to bind. diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs index 138157469d..0034f47043 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs @@ -8141,6 +8141,139 @@ fn graph_off_stale_witness_reconciles_unchanged_source_without_full_decode() { ); } +/// The disk freshness witness names whichever generation last persisted it. +/// A later seal of the same bytes used to return `None` the moment that id +/// disagreed, and the graph-on caller then decoded and resealed. Under load +/// that reseal outlived the admission window, the swap cleared the witness, +/// and the newer generation never became current. Unchanged sealed bytes +/// keep the generation and rewrite the witness onto it. Moved bytes still +/// refuse, without publishing a substitute. +#[test] +fn predecessor_freshness_witness_keeps_the_sealed_generation() { + let fixture = GitFixture::new(ALPHA_LIB_V1); + let store = TempDir::new().expect("store root"); + let mut scheduler = scheduler( + &fixture, + store.path().to_path_buf(), + Arc::new(SharedCodeIndexBytePoolV1::default()), + ); + let seeded = published(scheduler.reconcile_now().expect("seed retained generation")); + let metadata = scheduler + .servable_retained_text_generation() + .expect("publication store") + .expect("authenticated retained text generation") + .metadata() + .clone(); + let generation_id = metadata.manifest().generation_id.clone(); + let mut witness = + RestoreFreshnessWitnessV1::load(store.path()).expect("the seal persisted a proof"); + assert_eq!(witness.generation_id, generation_id.as_str()); + witness.generation_id = "generation.predecessor".to_owned(); + witness.persist(store.path()); + let index_path = fixture.path().join(".git/index"); + let index_mtime = std::fs::metadata(&index_path) + .expect("git index metadata") + .modified() + .expect("git index mtime"); + filetime::set_file_mtime( + &index_path, + filetime::FileTime::from_system_time(index_mtime + Duration::from_secs(2)), + ) + .expect("advance only the git index mtime"); + + let decodes_before = scheduler.sealed_decode_count(); + let outcome = scheduler + .reconcile_retained_text_generation_with(&metadata, false) + .expect("graph-on retained reconcile") + .expect("unchanged sealed bytes must not be dropped"); + let CodeIndexReconcileOutcomeV1::Noop(evidence) = outcome else { + panic!("predecessor proof must not reseal the same snapshot: {outcome:?}"); + }; + assert_eq!( + evidence.snapshot_content_identity, seeded.snapshot_content_identity, + "the noop names the generation that was already sealed" + ); + assert_eq!( + scheduler.sealed_decode_count(), + decodes_before, + "keeping the sealed generation must not decode it again" + ); + assert_eq!( + RestoreFreshnessWitnessV1::load(store.path()) + .expect("rebound proof") + .generation_id, + generation_id.as_str(), + "the disk proof must name the sealed generation, not the predecessor" + ); + assert_eq!( + scheduler + .source_currency_witness_for(&generation_id, &metadata.snapshot().content_identity,) + .map(|witness| witness.generation_id), + Some(generation_id.clone()), + "the in-memory proof must admit the sealed generation" + ); + + fixture.edit( + "src/lib.rs", + "pub fn changed_after_predecessor_proof() -> u32 { 2 }\n", + ); + let refused = scheduler + .reconcile_retained_text_generation_with(&metadata, false) + .expect("changed source is a typed refusal, not an error"); + assert!( + refused.is_none(), + "moved bytes must not keep the sealed generation: {refused:?}" + ); + assert_eq!( + scheduler + .publication + .read_publication_pointer() + .expect("read pointer") + .expect("active pointer") + .generation_id, + generation_id.as_str(), + "refusing the moved bytes must not publish a substitute generation" + ); +} + +/// Clone backfill and the seal itself outlive the 30s admission window. Expiry +/// is a request to re-check the sealed digests, not a reason to drop the +/// generation those digests already name. A byte change after expiry still drops it. +#[test] +fn expired_proof_keeps_the_sealed_generation_until_bytes_move() { + let fixture = GitFixture::new(ALPHA_LIB_V1); + let store = TempDir::new().expect("store root"); + let mut scheduler = scheduler( + &fixture, + store.path().to_path_buf(), + Arc::new(SharedCodeIndexBytePoolV1::default()), + ); + let seeded = published(scheduler.reconcile_now().expect("seed retained generation")); + scheduler.expire_source_proof_for_test(); + assert_eq!( + scheduler + .currency_witness_for_sealed_snapshot( + &seeded.generation_id, + &seeded.snapshot_content_identity, + ) + .map(|witness| witness.generation_id), + Some(seeded.generation_id.clone()), + "an expired proof must keep the generation whose sealed bytes still match" + ); + + fixture.edit("src/lib.rs", "pub fn alpha() -> u32 { 9 }\n"); + scheduler.expire_source_proof_for_test(); + assert!( + scheduler + .currency_witness_for_sealed_snapshot( + &seeded.generation_id, + &seeded.snapshot_content_identity, + ) + .is_none(), + "an expired proof must drop the generation once its sealed bytes moved" + ); +} + /// A query freshness probe against a restored owner that no pass has verified /// yet must report "not current", the restart's first pass is still the /// remedy, without minting an observed source change: no overflow hint and no From ed3d945b39ee3ce1e5a0594df12129f7e9f79ea5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:30:29 +0000 Subject: [PATCH 15/84] fix(lcm): describe the captured message, not a stub The registered describe path blanked every preview and treated the snippet length as the message size, so a session expand could already read came back empty. Report the bounded snippet and the stored message length instead. Co-authored-by: Zack Jackson --- .../src/registered_lcm_render.rs | 98 +++++++++++++++---- .../src/registered_lcm_render/tests.rs | 77 +++++++++++++++ .../mcp_suite/mcp_handler_test/lcm_test.rs | 17 +++- .../mcp_handler_test/session_search_test.rs | 33 +++++++ 4 files changed, 203 insertions(+), 22 deletions(-) diff --git a/crates/tracedecay-session-temporal-store/src/registered_lcm_render.rs b/crates/tracedecay-session-temporal-store/src/registered_lcm_render.rs index 81c24492ca..a1c09679a3 100644 --- a/crates/tracedecay-session-temporal-store/src/registered_lcm_render.rs +++ b/crates/tracedecay-session-temporal-store/src/registered_lcm_render.rs @@ -304,40 +304,64 @@ async fn raw_message_overviews( provider: &str, session_id: &str, ) -> Result, LcmError> { + // The snippet is the bounded preview. `total_chars` is the message's own + // length: an external payload's recorded char count, otherwise the stored + // content. Using the snippet length here described a stub, which is how a + // session that expand can read came back empty. + let preview_cap = i64::try_from(tracedecay_lcm::MAX_DERIVED_SNIPPET_CHARS) + .map_err(|_| LcmError::Db("snippet preview cap does not fit i64".to_string()))?; let mut rows = query( snapshot, - "SELECT message_id, store_id, role, storage_kind, payload_ref, - LENGTH(snippet_text) - FROM lcm_raw_messages - WHERE provider = ?1 AND session_id = ?2 - ORDER BY store_id + "SELECT raw.message_id, raw.store_id, raw.role, raw.storage_kind, raw.payload_ref, + CASE + WHEN raw.snippet_text <> '' THEN raw.snippet_text + ELSE substr(COALESCE(raw.content, ''), 1, ?3) + END, + COALESCE( + (SELECT payload.char_count + FROM lcm_external_payloads AS payload + WHERE payload.payload_ref = raw.payload_ref), + length(raw.content), + length(raw.snippet_text), + 0 + ) + FROM lcm_raw_messages AS raw + WHERE raw.provider = ?1 AND raw.session_id = ?2 + ORDER BY raw.store_id LIMIT 20", - params![provider, session_id], + params![provider, session_id, preview_cap], ) .await?; let mut out = Vec::new(); while let Some(row) = next_row(&mut rows).await? { let storage_kind_text: String = field!(&row, 3)?; - let total_chars = field!(&row, 5, i64)?.max(0) as u64; + let content_preview: String = field!(&row, 5)?; + let total_chars = field!(&row, 6, i64)?.max(0) as u64; out.push(LcmRawMessageOverview { message_id: field!(&row, 0)?, store_id: field!(&row, 1)?, role: field!(&row, 2)?, storage_kind: storage_kind(&storage_kind_text)?, payload_ref: field!(&row, 4)?, - content_preview: String::new(), - content_range: LcmContentRange { - offset: 0, - limit: 0, - returned_chars: 0, - total_chars, - truncated: total_chars > 0, - }, + content_range: preview_range(&content_preview, total_chars), + content_preview, }); } Ok(out) } +fn preview_range(preview: &str, total_chars: u64) -> LcmContentRange { + let returned_chars = preview.chars().count() as u64; + let total_chars = total_chars.max(returned_chars); + LcmContentRange { + offset: 0, + limit: returned_chars, + returned_chars, + total_chars, + truncated: returned_chars < total_chars, + } +} + async fn summary_overviews( snapshot: &(impl QueryExecutor + ?Sized), provider: &str, @@ -346,7 +370,7 @@ async fn summary_overviews( ) -> Result, LcmError> { let mut rows = query( snapshot, - "SELECT node_id, conversation_id, depth, created_at + "SELECT node_id, conversation_id, depth, summary_text, created_at FROM lcm_summary_nodes WHERE provider = ?1 AND session_id = ?2 ORDER BY depth, created_at, node_id @@ -357,14 +381,17 @@ async fn summary_overviews( let mut out = Vec::new(); while let Some(row) = next_row(&mut rows).await? { let node_id: String = field!(&row, 0)?; + let summary_text: String = field!(&row, 3)?; let source_count = relation(relations, &node_id)?.sources.len(); out.push(LcmSummaryNodeOverview { node_id, conversation_id: field!(&row, 1)?, depth: field!(&row, 2)?, - summary_preview: String::new(), + summary_preview: tracedecay_lcm::retrieval_content::derived_text_for_snippet( + &summary_text, + ), source_count, - created_at: field!(&row, 3)?, + created_at: field!(&row, 4)?, }); } Ok(out) @@ -489,6 +516,14 @@ async fn describe_external_payload( if payload.provider != provider || payload.session_id != session_id { return Err(LcmError::PayloadNotFound); } + let content_preview = external_payload_preview( + snapshot, + provider, + session_id, + &payload.message_id, + payload_ref, + ) + .await?; Ok(LcmDescribeExternalPayload { payload_ref: payload.payload_ref, provider: payload.provider, @@ -500,10 +535,35 @@ async fn describe_external_payload( char_count: payload.char_count, created_at: payload.created_at, metadata_json: payload.metadata_json, - content_preview: String::new(), + content_preview, }) } +async fn external_payload_preview( + snapshot: &(impl QueryExecutor + ?Sized), + provider: &str, + session_id: &str, + message_id: &str, + payload_ref: &str, +) -> Result { + let mut rows = query( + snapshot, + "SELECT snippet_text + FROM lcm_raw_messages + WHERE provider = ?1 + AND session_id = ?2 + AND message_id = ?3 + AND payload_ref = ?4 + LIMIT 1", + params![provider, session_id, message_id, payload_ref], + ) + .await?; + if let Some(row) = next_row(&mut rows).await? { + return field!(&row, 0); + } + Ok(format!("[externalized payload ref={payload_ref}]")) +} + /// Loads the raw row a directly requested `store_id` names, refusing when it is /// gone. Summary *lineage* reads must use [`find_raw_message`] instead: an /// absent row there is retention, not a missing target. diff --git a/crates/tracedecay-session-temporal-store/src/registered_lcm_render/tests.rs b/crates/tracedecay-session-temporal-store/src/registered_lcm_render/tests.rs index 426598ee66..2dfcd7faec 100644 --- a/crates/tracedecay-session-temporal-store/src/registered_lcm_render/tests.rs +++ b/crates/tracedecay-session-temporal-store/src/registered_lcm_render/tests.rs @@ -393,3 +393,80 @@ async fn registered_metadata_rows_do_not_fabricate_full_raw_messages() { "expected a payload-integrity refusal, got: {error:?}" ); } + +#[tokio::test] +async fn session_describe_reports_the_message_not_an_empty_stub() { + let directory = tempdir().expect("temporary session store"); + let runtime = seeded_render_fixture(directory.path()).await; + let content = "canonical raw message plus hidden tail"; + runtime + .registered_database(HostAdmissionScope::Profile) + .expect("registered session database") + .writer_connection() + .expect("registered writer") + .execute_batch(&format!( + "UPDATE lcm_raw_messages + SET content = '{content}', snippet_text = 'canonical raw' + WHERE message_id = 'message-a';" + )) + .await + .expect("shorten the stored preview without shortening the message"); + let snapshot = runtime + .registered_database(HostAdmissionScope::Profile) + .expect("registered session database") + .read_snapshot() + .await + .expect("registered read snapshot"); + + let description = describe( + &snapshot, + LcmDescribeRequest { + provider: "codex".to_string(), + session_id: "session-a".to_string(), + target: LcmDescribeTarget::Session, + }, + &canonical_fixture_relations(), + ) + .await + .expect("session describe"); + + let overview = description + .raw_messages + .iter() + .find(|message| message.message_id == "message-a") + .expect("describe must list the captured message"); + assert_eq!(overview.content_preview, "canonical raw"); + assert!(!overview.content_preview.contains("hidden tail")); + assert_eq!( + overview.content_range.total_chars, + content.chars().count() as u64 + ); + assert!(overview.content_range.truncated); + let summary = description + .summary_nodes + .iter() + .find(|node| node.node_id == "summary-child") + .expect("describe must list the summary"); + assert_eq!(summary.summary_preview, "canonical child summary"); + + let payload = describe( + &snapshot, + LcmDescribeRequest { + provider: "codex".to_string(), + session_id: "session-a".to_string(), + target: LcmDescribeTarget::ExternalPayload { + payload_ref: "payload-a".to_string(), + }, + }, + &[], + ) + .await + .expect("external payload describe"); + assert_eq!( + payload + .external_payload + .expect("payload metadata") + .content_preview, + "canonical external payload" + ); +} diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_test.rs index a39c62135b..6f6112c323 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_test.rs @@ -286,10 +286,21 @@ async fn lcm_session_handlers_expose_bounded_read_apis_and_placeholders() { "{described_payload}" ); assert_eq!(described_payload["description"]["raw_message_count"], 1); + let preview = described_payload["description"]["raw_messages"][0]["content_preview"] + .as_str() + .expect("describe preview"); assert!( - described_payload["description"]["raw_messages"][0] - .get("content_preview") - .is_some() + preview.starts_with("orchard dispatch"), + "describe returned an empty preview: {preview:?}" + ); + assert!( + preview.chars().count() < full_text.chars().count(), + "describe echoed the full payload body" + ); + assert_eq!( + described_payload["description"]["raw_messages"][0]["content_range"]["total_chars"], + full_text.chars().count() as u64, + "describe must name the captured message length, not the preview stub" ); assert!( described_payload["description"]["raw_messages"][0] diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/session_search_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/session_search_test.rs index 2d409392bd..145885dce7 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/session_search_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/session_search_test.rs @@ -649,6 +649,39 @@ async fn production_codex_hook_ingest_survives_message_search_reopen() { expanded["expansion"]["raw_message"]["message_id"], message_id, "{expanded}" ); + let described = call_production_tool( + &harness, + &project, + "tracedecay_lcm_describe", + json!({ + "provider": "codex", + "session_id": session_id, + "target": {"kind": "session"}, + "format": "json" + }), + ) + .await; + let captured = "Find the cobalt orchard scheduler migration"; + let overview = described["description"]["raw_messages"] + .as_array() + .and_then(|messages| { + messages + .iter() + .find(|message| message["message_id"] == message_id) + }) + .unwrap_or_else(|| panic!("describe omitted the captured prompt: {described}")); + assert_eq!( + overview["content_range"]["total_chars"], + captured.chars().count() as u64, + "{overview}" + ); + let preview = overview["content_preview"] + .as_str() + .unwrap_or_else(|| panic!("describe preview missing: {overview}")); + assert!( + preview.contains("cobalt orchard"), + "describe preview was empty: {preview:?}" + ); harness.shutdown().await; From 6099dda2c4acc73a09b4e2f8516a399455acbe0b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:31:40 +0000 Subject: [PATCH 16/84] fix(daemon): close socket-group and pointer-rename races Harness stop signaled only the detached group leader, so descendants kept the listen socket after wait. Publication renamed the active pointer without checking the bytes it had observed. Signal the whole group before reap, unlink the published socket, and refuse a pointer rename whose file is no longer that observation. Co-authored-by: Zack Jackson --- .../code_index_generations/text_artifacts.rs | 9 ++ .../code_index_scheduler/publication_store.rs | 129 ++++++++++++----- .../tests/publication_store.rs | 55 +++++++ crates/tracedecay/tests/common/mod.rs | 137 ++++++++++++------ .../daemon_fixture.rs | 2 + .../flight_tests.rs | 5 + crates/tracedecay/tests/daemon_suite/main.rs | 1 + .../daemon_suite/socket_lifecycle_test.rs | 89 ++++++++++++ 8 files changed, 341 insertions(+), 86 deletions(-) create mode 100644 crates/tracedecay/tests/daemon_suite/socket_lifecycle_test.rs diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/text_artifacts.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/text_artifacts.rs index c62e73dc92..170e69a3ff 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/text_artifacts.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/text_artifacts.rs @@ -198,6 +198,15 @@ fn mutate_verified_text_artifact_under_lock( "publication pointer exceeds its durable byte bound".to_owned(), )); } + // Re-read immediately before the rename. A pointer that is no longer the + // one this mutation observed — including a truncated file — must not be + // replaced by the in-memory copy. + let current = read_active_pointer(store_root)?; + if ¤t != expected_pointer { + return Err(CodeGenerationRetentionErrorV1::Conflict( + "active generation pointer changed before text-artifact mutation".to_owned(), + )); + } atomic_write( &store_root.join(ACTIVE_POINTER_FILE), "code-generation-text-artifact-mutation", diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs index 3422489f5e..d52e0c3962 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs @@ -1230,38 +1230,94 @@ impl DaemonCodeIndexPublicationStoreV1 { "durable code-generation index exceeds its retention bounds", )); } - *self + let mut memo = self .pointer_memo .lock() - .unwrap_or_else(PoisonError::into_inner) = Some(PublicationPointerMemoV1 { - mtime, - size, - digest, - pointer: pointer.clone(), - }); + .unwrap_or_else(PoisonError::into_inner); + // Install only when the file is still the bytes just parsed. A rename + // that landed during validation owns the memo. + if std::fs::read(&self.active_path).ok().as_deref() == Some(bytes.as_slice()) { + *memo = Some(PublicationPointerMemoV1 { + mtime, + size, + digest, + pointer: pointer.clone(), + }); + } Ok(Some(pointer)) } fn remember_publication_pointer(&self, pointer: &DurablePublicationPointerV1, bytes: &[u8]) { - let metadata = match std::fs::metadata(&self.active_path) { - Ok(metadata) => metadata, - Err(_) => { - *self - .pointer_memo - .lock() - .unwrap_or_else(PoisonError::into_inner) = None; - return; - } - }; - *self + let mut memo = self .pointer_memo .lock() - .unwrap_or_else(PoisonError::into_inner) = Some(PublicationPointerMemoV1 { - mtime: metadata.modified().ok(), - size: metadata.len(), - digest: Self::state_digest(bytes), - pointer: pointer.clone(), - }); + .unwrap_or_else(PoisonError::into_inner); + // The memo and the file it names are one critical section. A publisher + // that observed older bytes must not install them over a newer file. + match std::fs::read(&self.active_path) { + Ok(current) if current == bytes => { + let metadata = std::fs::metadata(&self.active_path).ok(); + *memo = Some(PublicationPointerMemoV1 { + mtime: metadata + .as_ref() + .and_then(|metadata| metadata.modified().ok()), + size: metadata.map(|metadata| metadata.len()).unwrap_or(0), + digest: Self::state_digest(bytes), + pointer: pointer.clone(), + }); + } + Ok(_) => {} + Err(_) => *memo = None, + } + } + + /// Replace the active pointer only when it is still the exact bytes this + /// publication observed under the store lock. + /// + /// `rename(2)` replaces whatever occupies the path, including a truncated + /// or rewritten pointer. The observation is the compare-and-swap token: + /// a mismatch is a refusal, not a rewrite. `lock` is the witness that + /// this critical section is the exclusive owner of the store. + pub(super) fn commit_observed_pointer( + &self, + _lock: &CodeGenerationStoreLockV1, + observed: Option<&[u8]>, + pointer: &DurablePublicationPointerV1, + bytes: &[u8], + ) -> Result<(), CodeIndexPublicationStoreErrorV1> { + let current = match std::fs::read(&self.active_path) { + Ok(current) => Some(current), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(Self::unavailable(error)), + }; + if current.as_deref() != observed { + return Err(match current { + Some(current) + if serde_json::from_slice::(¤t).is_err() => + { + Self::corruption("active code-generation pointer is corrupt") + } + _ => CodeIndexPublicationStoreErrorV1::CompareAndSwap, + }); + } + let temporary = self + .active_path + .with_extension(format!("json.{}.tmp", std::process::id())); + if temporary.exists() { + std::fs::remove_file(&temporary).map_err(Self::unavailable)?; + } + Self::write_durable(&temporary, bytes)?; + if let Err(error) = std::fs::rename(&temporary, &self.active_path) { + let _ = std::fs::remove_file(&temporary); + return Err(Self::unavailable(error)); + } + Self::sync_directory( + self.active_path + .parent() + .ok_or_else(|| Self::unavailable("active pointer has no parent directory"))?, + )?; + self.remember_publication_pointer(pointer, bytes); + Ok(()) } pub(super) fn read_retained_partitioned_segment( @@ -2174,6 +2230,15 @@ impl CodeIndexAtomicPublicationPort for DaemonCodeIndexPublicationStoreV1 { } else { self.read_publication_pointer()? }; + // The bytes behind `prior_pointer`, captured under the store lock. + // The commit below refuses to rename unless the file is still these + // exact bytes, so a pointer that changed after this observation is + // not overwritten. + let prior_bytes = if prior_pointer.is_some() { + Some(std::fs::read(&self.active_path).map_err(Self::unavailable)?) + } else { + None + }; if undecoded_expectation.is_none() && prior_pointer .as_ref() @@ -2551,22 +2616,8 @@ impl CodeIndexAtomicPublicationPort for DaemonCodeIndexPublicationStoreV1 { } else { None }; - let temporary = self - .active_path - .with_extension(format!("json.{}.tmp", std::process::id())); - if temporary.exists() { - std::fs::remove_file(&temporary).map_err(Self::unavailable)?; - } hotpath::measure_block!("code_index.generation.publish.pointer_commit", { - Self::write_durable(&temporary, &bytes)?; - std::fs::rename(&temporary, &self.active_path).map_err(Self::unavailable)?; - Self::sync_directory( - self.active_path - .parent() - .ok_or_else(|| Self::unavailable("active pointer has no parent directory"))?, - )?; - self.remember_publication_pointer(&pointer, &bytes); - Ok::<(), CodeIndexPublicationStoreErrorV1>(()) + self.commit_observed_pointer(&_store_lock, prior_bytes.as_deref(), &pointer, &bytes) })?; drop(source_fence); let mut state = self.cache.lock_state()?; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/publication_store.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/publication_store.rs index 95f3f7a4d7..f5285d04d0 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/publication_store.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/publication_store.rs @@ -2648,3 +2648,58 @@ fn publication_pointer_memo_follows_bytes_when_size_and_mtime_stay_put() { "equal size and mtime must not reuse the previous pointer" ); } + +#[test] +fn stale_pointer_commit_does_not_replace_a_changed_active_pointer() { + let store = TempDir::new().expect("store root"); + let project = TempDir::new().expect("project root"); + let publication = super::super::DaemonCodeIndexPublicationStoreV1::new( + store.path(), + project.path(), + SanitizerRevision::new(tracedecay_privacy::CODE_SOURCE_SANITIZER_VERSION_V1) + .expect("sanitizer revision"), + ) + .expect("open publication store"); + let pointer_path = store.path().join("active-code-generation-v1.json"); + let observed = same_length_publication_pointer("generation.observed", 0x31); + let observed_bytes = serde_json::to_vec(&observed).expect("encode observed pointer"); + let replacement = same_length_publication_pointer("generation.replacement", 0x32); + let replacement_bytes = serde_json::to_vec(&replacement).expect("encode replacement pointer"); + std::fs::write(&pointer_path, &observed_bytes).expect("write observed pointer"); + let store_lock = acquire_code_generation_store_lock(store.path()).expect("store lock"); + + publication + .commit_observed_pointer( + &store_lock, + Some(&observed_bytes), + &replacement, + &replacement_bytes, + ) + .expect("matching observation publishes"); + assert_eq!( + std::fs::read(&pointer_path).expect("published pointer"), + replacement_bytes + ); + + std::fs::write(&pointer_path, b"{").expect("truncate active pointer"); + let error = publication + .commit_observed_pointer( + &store_lock, + Some(&replacement_bytes), + &observed, + &observed_bytes, + ) + .expect_err("a stale observation must not publish"); + assert!( + matches!( + error, + CodeIndexPublicationStoreErrorV1::CorruptionResetRequired(_) + ), + "corrupt pointer is a closed publication failure, not a rewrite: {error:?}" + ); + assert_eq!( + std::fs::read(&pointer_path).expect("faulted pointer remains"), + b"{", + "the truncated pointer must still be the file" + ); +} diff --git a/crates/tracedecay/tests/common/mod.rs b/crates/tracedecay/tests/common/mod.rs index 28bbc596c4..97f60f8423 100644 --- a/crates/tracedecay/tests/common/mod.rs +++ b/crates/tracedecay/tests/common/mod.rs @@ -657,6 +657,11 @@ pub fn http_agent_with_timeout(timeout: Duration) -> ureq::Agent { /// panic while the child is still running, `Drop` force-stops and reaps it. pub struct TestChildProcess { child: Child, + /// Path of a Unix socket this child published. Released after the process + /// group is reaped so a descendant that still holds the listen descriptor + /// cannot keep the path accepting. + #[cfg(unix)] + release_socket: Option, } /// Daemon-specific name retained for test fixtures that keep a daemon alive. @@ -664,7 +669,38 @@ pub type DaemonProcess = TestChildProcess; impl TestChildProcess { pub fn new(child: Child) -> Self { - Self { child } + Self { + child, + #[cfg(unix)] + release_socket: None, + } + } + + /// Unlink `path` once this child has been reaped. + /// + /// `process_group(0)` makes the child a group leader. Stopping only that + /// pid leaves descendants that still hold the listen socket. Group-kill + /// closes those descriptors; unlinking the path is what makes a later + /// `connect` fail even if the kernel has not finished the last close. + #[cfg(unix)] + pub fn release_socket_on_stop(&mut self, path: PathBuf) { + self.release_socket = Some(path); + } + + #[cfg(unix)] + fn release_recorded_socket(&mut self) { + if let Some(path) = self.release_socket.take() { + match std::fs::remove_file(&path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + panic!( + "failed to release daemon socket '{}': {error}", + path.display() + ) + } + } + } } pub fn id(&self) -> u32 { @@ -752,9 +788,14 @@ impl TestChildProcess { /// Force-stops the daemon and reaps its process before returning. /// /// `Child::kill` maps to `SIGKILL` on Unix and the platform termination - /// primitive elsewhere, keeping fault-injection tests portable. + /// primitive elsewhere, keeping fault-injection tests portable. On Unix + /// the child's process group is signaled first, then the published socket + /// path is unlinked. pub fn kill_and_wait(&mut self) -> std::io::Result { - terminate_and_reap(&mut self.child) + let status = terminate_and_reap(&mut self.child)?; + #[cfg(unix)] + self.release_recorded_socket(); + Ok(status) } fn drain_stderr(&mut self) { @@ -779,11 +820,25 @@ impl TestChildProcess { impl Drop for TestChildProcess { fn drop(&mut self) { let _ = terminate_and_reap(&mut self.child); + #[cfg(unix)] + self.release_recorded_socket(); } } -/// PID-directed stop: survives `process_group(0)` / `setsid` detachment. +/// Stop a child that was detached with `process_group(0)`. +/// +/// The child is the leader of its own group. `SIGKILL` of that pid alone +/// leaves descendants in the group. Those descendants keep any descriptor they +/// inherited, including a listen socket, so the path stays connectable after +/// `wait` returns. Signaling the group first closes those descriptors; the +/// leader kill still covers a child whose `setpgid` has not run yet. fn terminate_and_reap(child: &mut Child) -> std::io::Result { + // Signal the group before reaping. A leader that has already exited still + // names the group; returning on `try_wait` first would leave descendants + // holding the listen socket. + #[cfg(unix)] + signal_child_process_group(child.id()); + if let Ok(Some(status)) = child.try_wait() { return Ok(status); } @@ -798,6 +853,21 @@ fn terminate_and_reap(child: &mut Child) -> std::io::Result { child.wait() } +#[cfg(unix)] +fn signal_child_process_group(pid: u32) { + let Ok(pid) = i32::try_from(pid) else { + return; + }; + if pid == 0 { + return; + } + // SAFETY: `pid` is the spawned child's id. Negating it addresses the + // process group `process_group(0)` created with that pid as leader. + // `ESRCH` is ignored: the child may not be a group leader, and the pid + // kill in `terminate_and_reap` still stops it. + let _ = unsafe { libc::kill(-pid, libc::SIGKILL) }; +} + /// Detach a test child from the test process group. /// /// Nextest (and other harness timeouts) signal the test's process group. @@ -1089,13 +1159,6 @@ pub fn spawn_tracedecay_daemon_with( spawn_tracedecay_daemon_process(&home, &binary, configure) } -/// How long a replacement daemon waits for a stopped predecessor's endpoint to -/// stop accepting before reporting it as still live. -/// -/// Generous on purpose: the wait only costs time when a predecessor is -/// genuinely still reachable, and a real leak still fails rather than hangs. -const PREDECESSOR_DAEMON_VACATE_TIMEOUT: Duration = Duration::from_secs(10); - fn spawn_tracedecay_daemon_process( home: &Path, binary: &Path, @@ -1118,42 +1181,20 @@ fn spawn_tracedecay_daemon_process( }) .is_some_and(|address| TcpStream::connect(address).is_ok()) }; - // Stopping a predecessor daemon is asynchronous with respect to its - // endpoint: `kill` plus `wait` reaps the PID the harness spawned, but the - // kernel keeps the listening socket alive while *any* duplicate of that - // descriptor survives, including one a subprocess inherited across `fork` - // and still holds because it has not reached its own `exec` yet. Asserting - // instantaneously therefore reports an ordinary teardown tail as a live - // daemon, which is what `init_project_fixture` journeys (spawn, init, drop, - // spawn again) hit on a loaded runner. Wait a bounded time for the endpoint - // to stop accepting; a daemon that keeps accepting still fails with the - // same refusal. - poll_until( - Instant::now() + PREDECESSOR_DAEMON_VACATE_TIMEOUT, - Duration::from_millis(25), - || { - #[cfg(unix)] - let live = std::os::unix::net::UnixStream::connect(&socket_path).is_ok(); - #[cfg(not(unix))] - let live = portable_daemon_connectable(); - (!live).then_some(()) - }, - || { - #[cfg(unix)] - { - format!( - "refusing to replace a live test daemon at {}", - socket_path.display() - ) - } - #[cfg(not(unix))] - { - format!( - "refusing to replace a live test daemon recorded at {}", - authority_path.display() - ) - } - }, + // A predecessor stopped through this harness has already had its process + // group reaped and its socket path unlinked. A path that still accepts is + // a daemon this spawn does not own. + #[cfg(unix)] + assert!( + std::os::unix::net::UnixStream::connect(&socket_path).is_err(), + "refusing to replace a live test daemon at {}", + socket_path.display() + ); + #[cfg(not(unix))] + assert!( + !portable_daemon_connectable(), + "refusing to replace a live test daemon recorded at {}", + authority_path.display() ); let mut command = Command::new(binary); @@ -1169,6 +1210,8 @@ fn spawn_tracedecay_daemon_process( detach_from_test_process_group(&mut command); let child = command.spawn().expect("tracedecay daemon should start"); let mut daemon = DaemonProcess::new(child); + #[cfg(unix)] + daemon.release_socket_on_stop(socket_path.clone()); let deadline = Instant::now() + Duration::from_secs(10); poll_until( diff --git a/crates/tracedecay/tests/daemon_suite/advanced_workflow_journey/daemon_fixture.rs b/crates/tracedecay/tests/daemon_suite/advanced_workflow_journey/daemon_fixture.rs index d0bbc79f28..069897ebb7 100644 --- a/crates/tracedecay/tests/daemon_suite/advanced_workflow_journey/daemon_fixture.rs +++ b/crates/tracedecay/tests/daemon_suite/advanced_workflow_journey/daemon_fixture.rs @@ -105,6 +105,8 @@ pub(super) fn spawn_project_daemon(home: &Path, project: &Path) -> common::Daemo .spawn() .expect("advanced workflow daemon should start"); let mut daemon = common::DaemonProcess::new(child); + #[cfg(unix)] + daemon.release_socket_on_stop(common::daemon_socket_path(home)); let daemon_pid = u64::from(daemon.id()); let deadline = Instant::now() + Duration::from_secs(120); loop { diff --git a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs index bc1388df7f..e679f5855a 100644 --- a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs +++ b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs @@ -331,6 +331,11 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { .expect_err("follower publication fails closed"); assert_publication_error(owner_error); assert_publication_error(follower_error); + assert_eq!( + std::fs::read(&pointer_path).expect("faulted pointer remains"), + b"{", + "publication must not replace a pointer it did not observe" + ); std::fs::write(pointer_path, pointer_bytes).expect("restore active pointer"); registry.shutdown().await; diff --git a/crates/tracedecay/tests/daemon_suite/main.rs b/crates/tracedecay/tests/daemon_suite/main.rs index 6339fab5c6..8e86e9cd8d 100644 --- a/crates/tracedecay/tests/daemon_suite/main.rs +++ b/crates/tracedecay/tests/daemon_suite/main.rs @@ -31,6 +31,7 @@ mod indexing_lifecycle_test; mod invocation_observability; mod invocation_primitives; #[cfg(unix)] +mod socket_lifecycle_test; #[cfg(unix)] mod stale_client_resilience_test; mod workflow_handoff_test; diff --git a/crates/tracedecay/tests/daemon_suite/socket_lifecycle_test.rs b/crates/tracedecay/tests/daemon_suite/socket_lifecycle_test.rs new file mode 100644 index 0000000000..aa1eed3648 --- /dev/null +++ b/crates/tracedecay/tests/daemon_suite/socket_lifecycle_test.rs @@ -0,0 +1,89 @@ +//! Process-group stop must release a listen socket held by a descendant. +//! +//! `process_group(0)` makes the spawned child its own group leader. Killing +//! only that pid leaves the descendant that inherited the listen descriptor, +//! and the path stays connectable after `wait` returns. + +use std::os::unix::net::UnixStream; +use std::os::unix::process::CommandExt; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +use crate::common::TestChildProcess; + +const HOLDER: &str = r#" +import os, socket, time +path = os.environ["TRACEDECAY_TEST_SOCKET_PATH"] +listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) +listener.bind(path) +listener.listen(1) +os.fork() +while True: + time.sleep(60) +"#; + +#[test] +fn group_stop_releases_an_inherited_listen_socket() { + let scratch = tempfile::tempdir().expect("socket scratch"); + let socket = scratch.path().join("daemon.sock"); + let mut command = Command::new("python3"); + command + .arg("-c") + .arg(HOLDER) + .env("TRACEDECAY_TEST_SOCKET_PATH", &socket) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + command.process_group(0); + let child = command.spawn().expect("spawn socket holder"); + // Do not record a socket path: connect must fail because the group is + // dead, not because the path was unlinked. + let mut holder = TestChildProcess::new(child); + + let ready_deadline = Instant::now() + Duration::from_secs(5); + while UnixStream::connect(&socket).is_err() { + assert!( + Instant::now() < ready_deadline, + "holder did not bind {}", + socket.display() + ); + if holder.try_wait().expect("holder status").is_some() { + panic!("socket holder exited before binding"); + } + std::thread::sleep(Duration::from_millis(20)); + } + + holder.kill_and_wait().expect("reap socket holder group"); + assert!( + socket.exists(), + "this proof must not delete the socket path" + ); + let refused = UnixStream::connect(&socket); + assert!( + refused.is_err(), + "process-group stop must release the inherited listen socket, connect returned {refused:?}" + ); +} + +#[test] +fn stop_unlinks_the_socket_path_the_child_published() { + let scratch = tempfile::tempdir().expect("socket scratch"); + let socket = scratch.path().join("daemon.sock"); + std::os::unix::net::UnixListener::bind(&socket).expect("bind socket"); + let mut command = Command::new("python3"); + command + .arg("-c") + .arg("import time; time.sleep(60)") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + command.process_group(0); + let child = command.spawn().expect("spawn sleeper"); + let mut sleeper = TestChildProcess::new(child); + sleeper.release_socket_on_stop(socket.clone()); + drop(sleeper); + assert!( + !socket.exists(), + "stopping the child must unlink the socket path it published" + ); +} From 0aed193ef6b24c0fa17da647c209205aeb3e5d44 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:32:13 +0000 Subject: [PATCH 17/84] fix(memory): treat memory.max as the service ceiling Host reserve and the cgroup ceiling are alternative protections, not stacked discounts. memory.max is the hard ceiling; memory.high below it is the reclaim watermark. A 128 GiB host with max=30 GiB and high=26 GiB stays open at 24 GiB RSS instead of carving another margin out of high. Co-authored-by: Zack Jackson --- .../src/resident_memory.rs | 187 +++++++++++++----- .../src/resident_memory/tests.rs | 160 +++++++++++++-- 2 files changed, 275 insertions(+), 72 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/resident_memory.rs b/crates/tracedecay-runtime-core/src/resident_memory.rs index f1669a488a..a40c4d5b50 100644 --- a/crates/tracedecay-runtime-core/src/resident_memory.rs +++ b/crates/tracedecay-runtime-core/src/resident_memory.rs @@ -23,8 +23,8 @@ pub const DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1: NonZeroU64 = /// Environment override for the process resident-memory admission limit, in /// bytes. Unset, unparseable, or zero values fall back to the RAM-derived /// authority. The code-index worker pool derives its reservation from this -/// same limit, so raising it can both admit and widen indexing, up to any -/// finite cgroup-v2 memory ceiling. +/// same limit, so raising it can both admit and widen indexing, up to the +/// hard cgroup ceiling (`memory.max`, or `memory.high` when max is unlimited). pub const PROCESS_RESIDENT_MEMORY_LIMIT_ENV_V1: &str = "TRACEDECAY_RESIDENT_MEMORY_LIMIT_BYTES"; const PROC_SELF_CGROUP_V1: &str = "/proc/self/cgroup"; @@ -85,15 +85,41 @@ fn finite_cgroup_memory_value_v1(path: &Path) -> Option { value.parse::().ok().map(|value| value.max(1)) } -fn cgroup_v2_memory_limit_v1(proc_self_cgroup: &Path, cgroup_root: &Path) -> Option { +/// The two cgroup-v2 memory controls on this process, walked to the mount root. +/// +/// `memory.max` is the kernel kill line. `memory.high` is the reclaim line +/// underneath it. They stay separate so each is used for what it is. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct CgroupMemoryCeilingV1 { + max_bytes: Option, + high_bytes: Option, +} + +/// Hard service ceiling: `memory.max` when it is finite, otherwise `memory.high`. +/// +/// A lone `memory.high` is the ceiling because the operator left no band above +/// the reclaim line. When both are finite, high is pressure, not a tighter max. +fn cgroup_service_ceiling_bytes(ceiling: CgroupMemoryCeilingV1) -> Option { + ceiling.max_bytes.or(ceiling.high_bytes) +} + +fn tighten(bound: Option, limit: u64) -> u64 { + bound.map_or(limit, |current| current.min(limit)) +} + +fn cgroup_v2_memory_ceiling_v1( + proc_self_cgroup: &Path, + cgroup_root: &Path, +) -> Option { let mut directory = cgroup_v2_process_directory_v1(proc_self_cgroup, cgroup_root)?; - let mut effective_limit = None; + let mut max_bytes = None; + let mut high_bytes = None; loop { - for filename in ["memory.max", "memory.high"] { - if let Some(limit) = finite_cgroup_memory_value_v1(&directory.join(filename)) { - effective_limit = - Some(effective_limit.map_or(limit, |current: u64| current.min(limit))); - } + if let Some(limit) = finite_cgroup_memory_value_v1(&directory.join("memory.max")) { + max_bytes = Some(tighten(max_bytes, limit)); + } + if let Some(limit) = finite_cgroup_memory_value_v1(&directory.join("memory.high")) { + high_bytes = Some(tighten(high_bytes, limit)); } if directory == cgroup_root { break; @@ -104,7 +130,10 @@ fn cgroup_v2_memory_limit_v1(proc_self_cgroup: &Path, cgroup_root: &Path) -> Opt } directory = parent.to_path_buf(); } - effective_limit + Some(CgroupMemoryCeilingV1 { + max_bytes, + high_bytes, + }) } fn effective_memory_bytes_v1(total_memory_bytes: u64, cgroup_limit: Option) -> u64 { @@ -115,68 +144,99 @@ fn effective_memory_bytes_v1(total_memory_bytes: u64, cgroup_limit: Option) } } -fn process_resident_memory_limit_v1( +struct ResidentMemoryAuthorityV1 { + limit_bytes: NonZeroU64, + /// `memory.high` when it sits strictly below the hard admission ceiling. + reclaim_watermark_bytes: Option, +} + +fn finite_nonzero_bytes(value: u64) -> NonZeroU64 { + NonZeroU64::new(value).unwrap_or(DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1) +} + +/// Admission ceiling for one host and one cgroup reading. +/// +/// Host reserve (one quarter of physical RAM) and the cgroup service ceiling +/// are alternative protections, not stacked discounts. The reserve applies +/// when the process can otherwise spend the machine. A finite cgroup already +/// reserved the rest of the machine, so the hard ceiling is +/// `min(host allowance, memory.max)` — or `memory.high` only when max is +/// unlimited. `memory.high` below that ceiling is the reclaim watermark, not +/// a second cut. An explicit override replaces the host reserve and is still +/// capped by the hard ceiling. +fn resident_memory_authority_v1( total_memory_bytes: u64, - cgroup_limit: Option, + cgroup: Option, override_limit: Option, -) -> NonZeroU64 { - // Retain one quarter of physical RAM for the OS and other processes, then - // respect the operator's cgroup ceiling as-is. Applying the quarter again - // *after* taking min(host, cgroup) double-discounted a deliberately sized - // service: 128 GiB host, memory.high=26 GiB became 19.5 GiB even though - // memory.max=30 GiB already retained the safety margin. An 18 GiB serving - // graph could then never admit its 2.6 GiB replacement builder. - let host_limit = (total_memory_bytes != 0) +) -> ResidentMemoryAuthorityV1 { + let cgroup = cgroup.unwrap_or(CgroupMemoryCeilingV1 { + max_bytes: None, + high_bytes: None, + }); + let service_ceiling = cgroup_service_ceiling_bytes(cgroup); + let host_allowance = (total_memory_bytes != 0) .then(|| process_resident_memory_limit_for_system_v1(total_memory_bytes)); - let automatic_limit = match (host_limit, cgroup_limit) { - (Some(host), Some(cgroup)) => NonZeroU64::new(host.get().min(cgroup)) - .unwrap_or(DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1), + let automatic_limit = match (host_allowance, service_ceiling) { + (Some(host), Some(ceiling)) => finite_nonzero_bytes(host.get().min(ceiling)), (Some(host), None) => host, - (None, Some(cgroup)) => { - NonZeroU64::new(cgroup).unwrap_or(DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1) - } + (None, Some(ceiling)) => finite_nonzero_bytes(ceiling), (None, None) => DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1, }; - override_limit.map_or(automatic_limit, |override_limit| { - cgroup_limit.map_or(override_limit, |cgroup_limit| { - NonZeroU64::new(override_limit.get().min(cgroup_limit)) - .unwrap_or(DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1) - }) - }) + let limit_bytes = match override_limit { + Some(override_limit) => match service_ceiling { + Some(ceiling) => finite_nonzero_bytes(override_limit.get().min(ceiling)), + None => override_limit, + }, + None => automatic_limit, + }; + let reclaim_watermark_bytes = cgroup.high_bytes.filter(|high| *high < limit_bytes.get()); + ResidentMemoryAuthorityV1 { + limit_bytes, + reclaim_watermark_bytes, + } } /// Size the shared resident-allocation authority for this process. /// -/// The automatic authority retains one quarter of physical RAM, then takes the -/// lower of that host allowance and this process's finite cgroup-v2 -/// `memory.max` / `memory.high`. A cgroup is already an operator-sized service -/// allowance and is not discounted a second time. +/// The automatic authority is the lower of the host reserve and this +/// process's hard cgroup ceiling (`memory.max`, or `memory.high` when max is +/// unlimited). A finite `memory.high` below that ceiling is the pressure +/// watermark, not a further discount of the ceiling. /// [`PROCESS_RESIDENT_MEMORY_LIMIT_ENV_V1`] can lower or raise the automatic -/// authority, but a finite cgroup ceiling remains an upper bound. The resulting -/// authority throttles simultaneous scratch ownership; it never limits -/// repository bytes on disk. +/// authority, but the hard cgroup ceiling remains an upper bound. The +/// resulting authority throttles simultaneous scratch ownership; it never +/// limits repository bytes on disk. #[must_use] pub fn detected_process_resident_memory_limit_v1() -> NonZeroU64 { + read_resident_memory_authority_v1().limit_bytes +} + +fn read_resident_memory_authority_v1() -> ResidentMemoryAuthorityV1 { let system = System::new_with_specifics( RefreshKind::new().with_memory(MemoryRefreshKind::new().with_ram()), ); let total_memory_bytes = system.total_memory(); let proc_self_cgroup = Path::new(PROC_SELF_CGROUP_V1); let cgroup_root = Path::new(CGROUP_V2_ROOT_V1); - let cgroup_limit = cgroup_v2_memory_limit_v1(proc_self_cgroup, cgroup_root); - let effective_memory_bytes = effective_memory_bytes_v1(total_memory_bytes, cgroup_limit); - let limit = process_resident_memory_limit_v1( + let cgroup = cgroup_v2_memory_ceiling_v1(proc_self_cgroup, cgroup_root); + let service_ceiling = cgroup.and_then(cgroup_service_ceiling_bytes); + let effective_memory_bytes = effective_memory_bytes_v1(total_memory_bytes, service_ceiling); + let authority = resident_memory_authority_v1( total_memory_bytes, - cgroup_limit, + cgroup, process_resident_memory_limit_override_v1(), ); hotpath::gauge!("resident_memory.system_total_bytes").set(total_memory_bytes as f64); hotpath::gauge!("resident_memory.effective_total_bytes").set(effective_memory_bytes as f64); - if let Some(cgroup_limit) = cgroup_limit { - hotpath::gauge!("resident_memory.cgroup_limit_bytes").set(cgroup_limit as f64); + if let Some(high_bytes) = cgroup.and_then(|ceiling| ceiling.high_bytes) { + hotpath::gauge!("resident_memory.cgroup_high_bytes").set(high_bytes as f64); + } + if let Some(service_ceiling) = service_ceiling { + hotpath::gauge!("resident_memory.cgroup_limit_bytes").set(service_ceiling as f64); } - hotpath::gauge!("resident_memory.admission_limit_bytes").set(limit.get() as f64); - limit + hotpath::gauge!("resident_memory.admission_limit_bytes") + .set(authority.limit_bytes.get() as f64); + authority } /// Fraction of the configured limit, in permille, at or above which *measured* @@ -343,15 +403,36 @@ impl fmt::Debug for ResidentMemoryPressureV1 { impl ResidentMemoryPressureV1 { #[must_use] pub fn new(limit_bytes: NonZeroU64) -> Self { - let high_watermark_bytes = resident_memory_watermark_bytes_v1( + Self::with_reclaim_line(limit_bytes, None) + } + + /// `reclaim_watermark_bytes` is a cgroup `memory.high` that sits strictly + /// below `limit_bytes`. It replaces the percentage high watermark so the + /// operator's band down to `memory.max` is not discounted again. Absent, + /// zero, or not strictly below the ceiling, the percentage watermarks stand. + fn with_reclaim_line(limit_bytes: NonZeroU64, reclaim_watermark_bytes: Option) -> Self { + let percentage_high = resident_memory_watermark_bytes_v1( limit_bytes, RESIDENT_MEMORY_PRESSURE_HIGH_WATERMARK_PERMILLE_V1, ); - let low_watermark_bytes = resident_memory_watermark_bytes_v1( + let percentage_low = resident_memory_watermark_bytes_v1( limit_bytes, RESIDENT_MEMORY_PRESSURE_LOW_WATERMARK_PERMILLE_V1, ) - .min(high_watermark_bytes); + .min(percentage_high); + let (high_watermark_bytes, low_watermark_bytes) = match reclaim_watermark_bytes { + Some(reclaim) if reclaim > 0 && reclaim < limit_bytes.get() => { + let low = u64::try_from( + u128::from(reclaim) + * u128::from(RESIDENT_MEMORY_PRESSURE_LOW_WATERMARK_PERMILLE_V1) + / u128::from(RESIDENT_MEMORY_PRESSURE_HIGH_WATERMARK_PERMILLE_V1), + ) + .unwrap_or(u64::MAX) + .min(reclaim); + (reclaim, low) + } + _ => (percentage_high, percentage_low), + }; Self { limit_bytes, high_watermark_bytes, @@ -547,8 +628,10 @@ static PROCESS_RESIDENT_MEMORY_PRESSURE_V1: OnceLock &'static Arc { PROCESS_RESIDENT_MEMORY_PRESSURE_V1.get_or_init(|| { - Arc::new(ResidentMemoryPressureV1::new( - detected_process_resident_memory_limit_v1(), + let authority = read_resident_memory_authority_v1(); + Arc::new(ResidentMemoryPressureV1::with_reclaim_line( + authority.limit_bytes, + authority.reclaim_watermark_bytes, )) }) } diff --git a/crates/tracedecay-runtime-core/src/resident_memory/tests.rs b/crates/tracedecay-runtime-core/src/resident_memory/tests.rs index 53bc42ca60..eeb78ea8b5 100644 --- a/crates/tracedecay-runtime-core/src/resident_memory/tests.rs +++ b/crates/tracedecay-runtime-core/src/resident_memory/tests.rs @@ -5,10 +5,11 @@ use std::sync::{Arc, Mutex, OnceLock}; use tracedecay_domain::{CodeGenerationId, ProjectId, WorktreeId}; use super::{ - ProcessResidentMemoryV1, RESIDENT_MEMORY_PRESSURE_ADMISSION_FLOOR_BYTES_V1, - ResidentMemoryAdmissionFailureV1, ResidentMemoryComponentIdV1, ResidentMemoryKeyV1, - ResidentMemoryPressureStateV1, ResidentMemoryPressureV1, cgroup_v2_memory_limit_v1, - effective_memory_bytes_v1, process_resident_memory_limit_v1, + CgroupMemoryCeilingV1, ProcessResidentMemoryV1, + RESIDENT_MEMORY_PRESSURE_ADMISSION_FLOOR_BYTES_V1, ResidentMemoryAdmissionFailureV1, + ResidentMemoryComponentIdV1, ResidentMemoryKeyV1, ResidentMemoryPressureStateV1, + ResidentMemoryPressureV1, cgroup_service_ceiling_bytes, cgroup_v2_memory_ceiling_v1, + effective_memory_bytes_v1, resident_memory_authority_v1, }; fn bytes(value: u64) -> NonZeroU64 { @@ -45,7 +46,8 @@ fn effective_memory_bytes( ) -> u64 { effective_memory_bytes_v1( total_memory_bytes, - cgroup_v2_memory_limit_v1(proc_self_cgroup, cgroup_root), + cgroup_v2_memory_ceiling_v1(proc_self_cgroup, cgroup_root) + .and_then(cgroup_service_ceiling_bytes), ) } @@ -115,30 +117,58 @@ fn root_v2_membership_reads_the_mount_root_ceiling() { ); } +fn hard_ceiling(max_bytes: u64) -> CgroupMemoryCeilingV1 { + CgroupMemoryCeilingV1 { + max_bytes: Some(max_bytes), + high_bytes: None, + } +} + #[test] fn configured_override_cannot_exceed_the_cgroup_ceiling() { let gib = 1024 * 1024 * 1024; + let capped = resident_memory_authority_v1( + 88 * gib, + Some(CgroupMemoryCeilingV1 { + max_bytes: Some(30 * gib), + high_bytes: Some(26 * gib), + }), + Some(bytes(64 * gib)), + ); assert_eq!( - process_resident_memory_limit_v1(88 * gib, Some(30 * gib), Some(bytes(64 * gib))).get(), - 30 * gib + capped.limit_bytes.get(), + 30 * gib, + "an override is capped by memory.max, not by the reclaim line" ); + assert_eq!(capped.reclaim_watermark_bytes, Some(26 * gib)); } #[test] fn cgroup_service_allowance_is_not_discounted_twice() { let gib = 1024 * 1024 * 1024; - + let only_high = resident_memory_authority_v1( + 128 * gib, + Some(CgroupMemoryCeilingV1 { + max_bytes: None, + high_bytes: Some(26 * gib), + }), + None, + ); assert_eq!( - process_resident_memory_limit_v1(128 * gib, Some(26 * gib), None).get(), + only_high.limit_bytes.get(), 26 * gib, - "the cgroup already reserves host headroom for this service" + "a lone memory.high is the service ceiling and is not quartered again" ); + assert_eq!(only_high.reclaim_watermark_bytes, None); + + let small_host = resident_memory_authority_v1(16 * gib, Some(hard_ceiling(30 * gib)), None); assert_eq!( - process_resident_memory_limit_v1(16 * gib, Some(30 * gib), None).get(), + small_host.limit_bytes.get(), 12 * gib, "a larger cgroup must not erase the physical-host reserve" ); + assert_eq!(small_host.reclaim_watermark_bytes, None); } #[test] @@ -155,7 +185,7 @@ fn unlimited_cgroup_memory_files_keep_host_memory_capacity() { } #[test] -fn finite_memory_high_below_max_is_the_effective_capacity() { +fn memory_high_does_not_replace_memory_max_as_the_hard_capacity() { let gib = 1024 * 1024 * 1024; let (_directory, proc_self_cgroup, cgroup_root) = cgroup_fixture( Some("0::/trace.slice/daemon.scope\n"), @@ -164,8 +194,13 @@ fn finite_memory_high_below_max_is_the_effective_capacity() { ); assert_eq!( effective_memory_bytes(88 * gib, &proc_self_cgroup, &cgroup_root), - 24 * gib + 30 * gib, + "memory.max is the kernel kill line" ); + let ceiling = cgroup_v2_memory_ceiling_v1(&proc_self_cgroup, &cgroup_root).expect("cgroup"); + let authority = resident_memory_authority_v1(88 * gib, Some(ceiling), None); + assert_eq!(authority.limit_bytes.get(), 30 * gib); + assert_eq!(authority.reclaim_watermark_bytes, Some(24 * gib)); } #[test] @@ -188,6 +223,82 @@ fn finite_ancestor_limit_bounds_an_unlimited_process_cgroup() { drop(directory); } +/// The slice owns `memory.max` and the service owns `memory.high`. +/// +/// On a 128 GiB host those are 30 GiB and 26 GiB. RSS at 24 GiB is still under +/// the reclaim line, and a 3 GiB replacement fits in the 4 GiB band down to it. +#[test] +fn slice_max_and_service_high_keep_the_reclaim_band_usable() { + let gib = 1024 * 1024 * 1024; + let (directory, proc_self_cgroup, cgroup_root) = cgroup_fixture( + Some("0::/trace.slice/daemon.scope\n"), + Some("max\n"), + Some(&format!("{}\n", 26 * gib)), + ); + fs::write( + cgroup_root.join("trace.slice/memory.max"), + format!("{}\n", 30 * gib), + ) + .expect("ancestor memory.max fixture"); + fs::write(cgroup_root.join("trace.slice/memory.high"), "max\n") + .expect("ancestor memory.high fixture"); + + let ceiling = cgroup_v2_memory_ceiling_v1(&proc_self_cgroup, &cgroup_root).expect("cgroup"); + let detected = resident_memory_authority_v1(128 * gib, Some(ceiling), None); + let pressure = Arc::new(ResidentMemoryPressureV1::with_reclaim_line( + detected.limit_bytes, + detected.reclaim_watermark_bytes, + )); + let authority = Arc::new(ProcessResidentMemoryV1::with_pressure( + detected.limit_bytes, + Arc::clone(&pressure), + )); + + assert_eq!(detected.limit_bytes.get(), 30 * gib); + assert_eq!(pressure.high_watermark_bytes(), 26 * gib); + assert_eq!( + pressure.low_watermark_bytes(), + 23_264_406_186, + "hysteresis stays at 750/900 of memory.high, not 75% of memory.max" + ); + assert!( + !pressure + .publish_observed_resident_bytes(22 * gib) + .is_over_budget() + ); + assert!( + !pressure + .publish_observed_resident_bytes(24 * gib) + .is_over_budget(), + "rss under memory.high is not over budget" + ); + authority + .reserve( + key("project-a", "worktree-a", "generation-a", "text-build"), + bytes(3 * gib), + ) + .expect("a 3 GiB replacement is admitted at 24 GiB RSS"); + + let headroom = detected + .limit_bytes + .get() + .saturating_sub(pressure.high_watermark_bytes()); + let available = detected + .limit_bytes + .get() + .saturating_sub(22 * gib) + .saturating_sub(headroom); + assert_eq!(available, 4 * gib); + assert!(available >= 1536 * 1024 * 1024); + + assert!( + pressure + .publish_observed_resident_bytes(26 * gib) + .is_over_budget() + ); + drop(directory); +} + #[test] fn low_effective_cgroup_ceiling_engages_measured_pressure_before_the_cap() { let mib = 1024 * 1024; @@ -196,17 +307,26 @@ fn low_effective_cgroup_ceiling_engages_measured_pressure_before_the_cap() { Some("134217728\n"), Some("100663296\n"), ); - let effective = effective_memory_bytes(8 * 1024 * mib, &proc_self_cgroup, &cgroup_root); - let limit = process_resident_memory_limit_v1(8 * 1024 * mib, Some(effective), None); - let pressure = Arc::new(ResidentMemoryPressureV1::new(limit)); + let ceiling = cgroup_v2_memory_ceiling_v1(&proc_self_cgroup, &cgroup_root).expect("cgroup"); + let detected = resident_memory_authority_v1(8 * 1024 * mib, Some(ceiling), None); + let pressure = Arc::new(ResidentMemoryPressureV1::with_reclaim_line( + detected.limit_bytes, + detected.reclaim_watermark_bytes, + )); let authority = Arc::new(ProcessResidentMemoryV1::with_pressure( - limit, + detected.limit_bytes, Arc::clone(&pressure), )); - assert_eq!(effective, 96 * mib); - assert_eq!(limit.get(), effective); - assert!(pressure.high_watermark_bytes() < effective); + assert_eq!(detected.limit_bytes.get(), 128 * mib); + assert_eq!(pressure.high_watermark_bytes(), 96 * mib); + assert!(pressure.high_watermark_bytes() < detected.limit_bytes.get()); + assert!( + !pressure + .publish_observed_resident_bytes(95 * mib) + .is_over_budget(), + "rss below memory.high is still under the hard ceiling" + ); assert!( pressure .publish_observed_resident_bytes(pressure.high_watermark_bytes()) From b63b775245c392c7adf80916f43ce72153c339b2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:33:46 +0000 Subject: [PATCH 18/84] fix(cli): report installed binary identity on update The GitHub upgrade path recorded the release tag as the daemon identity. The daemon advertises {release}+{sha}, and readiness compares those strings exactly, so update refused the binary it had just installed. Both install paths now record the binary's own --version and never substitute the tag. Co-authored-by: Zack Jackson --- crates/tracedecay-cli/src/upgrade.rs | 116 +++++++++++++++++++++------ 1 file changed, 90 insertions(+), 26 deletions(-) diff --git a/crates/tracedecay-cli/src/upgrade.rs b/crates/tracedecay-cli/src/upgrade.rs index 032cf81c0c..b93064d147 100644 --- a/crates/tracedecay-cli/src/upgrade.rs +++ b/crates/tracedecay-cli/src/upgrade.rs @@ -554,15 +554,15 @@ pub enum UpgradeOutcome { /// binary: `which_tracedecay()`'s current-exe-first order can point /// at the OLD binary (e.g. a stale Homebrew keg) after an upgrade. binary: Option, - /// Version of the freshly installed binary: the release-manifest - /// version for GitHub-release installs, the linked binary's - /// self-reported version for package-manager installs. Daemon restore - /// validates this version, the binary it actually restarts, instead - /// of the one that was running before the upgrade. `None` only when - /// the manager's install could not be interrogated; restore - /// verification then validates the pre-upgrade version and, if a new - /// daemon really was installed, fails with a typed identity mismatch - /// rather than silently passing. + /// Version the installed binary reports for itself (`--version`), + /// `{release}+{sha}[.dirty]`. Daemon restore compares this string to + /// the daemon's advertised build identity exactly, so a release tag + /// is not a substitute: the tag and the binary differ by build + /// metadata, and that mismatch is what failed `tracedecay update`'s + /// readiness wait. `None` only when the binary could not be + /// interrogated; restore then validates the pre-upgrade version and, + /// if a new daemon really was installed, fails with a typed identity + /// mismatch rather than accepting a less specific name. version: Option, }, /// Already on the latest version. The binary was not replaced. @@ -794,11 +794,20 @@ fn run_versioned_upgrade(current: &str, is_beta: bool) -> Result eprintln!("Upgrading v{current} → v{latest}..."); let binary = install_upgrade_version(latest, is_beta)?; record_previous_version(); - eprintln!("\x1b[32m✔\x1b[0m Successfully upgraded to v{latest}!"); - Ok(UpgradeOutcome::Installed { - binary, - version: Some(latest.to_owned()), - }) + Ok(finish_versioned_upgrade(latest, binary)) +} + +/// Completes a GitHub-release install. +/// +/// `catalog_version` is the release name shown to the operator. It is not +/// the installed identity: the published binary names itself +/// `{release}+{sha}` and the daemon advertises that same string. Readiness +/// compares the two exactly, so recording the catalog tag refuses the binary +/// this function just installed. +fn finish_versioned_upgrade(catalog_version: &str, binary: Option) -> UpgradeOutcome { + let version = probed_installed_version(binary.as_deref(), "installed release"); + eprintln!("\x1b[32m✔\x1b[0m Successfully upgraded to v{catalog_version}!"); + UpgradeOutcome::Installed { binary, version } } /// Atomically replaces `target` with the contents of `source`: the bytes are @@ -971,6 +980,28 @@ fn installed_binary_version_within( parse_version_output(&text).ok_or(VersionProbeError::Unrecognized(text)) } +/// The version `binary` reports for itself, or `None` when there is nothing +/// to ask or it does not answer. +/// +/// A missing answer is not filled in from a release tag. The tag omits the +/// commit the binary and the daemon both name, and readiness treats that +/// omission as a different identity. +fn probed_installed_version(binary: Option<&Path>, owner: &str) -> Option { + let Some(path) = binary else { + return None; + }; + match installed_binary_version(path) { + Ok(version) => Some(version), + Err(reason) => { + eprintln!( + " \x1b[33mwarning:\x1b[0m could not read the {owner} binary's version \ + ({reason}); daemon restore will not invent an identity" + ); + None + } + } +} + /// Whether a delegated manager upgrade was a no-op: the binary the manager /// links reports exactly the build version this process is running, which /// is the same file unless the manager installed something. `None` @@ -1024,17 +1055,8 @@ fn run_delegated_upgrade( None } }; - let installed_version = match binary.as_deref().map(installed_binary_version) { - Some(Ok(version)) => Some(version), - Some(Err(reason)) => { - eprintln!( - " \x1b[33mwarning:\x1b[0m could not read the {label}-installed binary's version \ - ({reason}); assuming a new install so the refresh chain runs" - ); - None - } - None => None, - }; + let installed_version = + probed_installed_version(binary.as_deref(), &format!("{label}-installed")); if delegated_upgrade_was_noop( crate::product_runtime::PRODUCT_BUILD_VERSION, installed_version.as_deref(), @@ -1234,7 +1256,8 @@ mod tests { use tracedecay_runtime_core::git::GitCommandError; use super::super::{ - VersionProbeError, installed_binary_version, installed_binary_version_within, + UpgradeOutcome, VersionProbeError, finish_versioned_upgrade, installed_binary_version, + installed_binary_version_within, }; fn script(dir: &Path, body: &str) -> PathBuf { @@ -1368,6 +1391,47 @@ mod tests { "a successful parent exit does not close an inherited pipe; the deadline must" ); } + + /// The observed update failure: GitHub names the release `0.1.0-beta.47` + /// and the binary that release ships names + /// `0.1.0-beta.47+`. Readiness compares those strings exactly, so + /// the catalog tag must not be the version the outcome records. + #[test] + fn a_release_install_reports_the_binary_identity_not_the_catalog_tag() { + let dir = tempfile::tempdir().unwrap(); + let sha = "84598a0b9c841b914565f46b20bb6c765706e8e5"; + let identity = format!("0.1.0-beta.47+{sha}"); + let binary = script(dir.path(), &format!("printf 'tracedecay {identity}\\n'")); + let catalog = "0.1.0-beta.47"; + + let outcome = finish_versioned_upgrade(catalog, Some(binary)); + + let UpgradeOutcome::Installed { version, .. } = outcome else { + panic!("a published release is an install, got {outcome:?}"); + }; + assert_eq!(version.as_deref(), Some(identity.as_str())); + assert_ne!( + version.as_deref(), + Some(catalog), + "the catalog tag is not the identity the daemon advertises" + ); + } + + /// A binary that cannot be asked must not be labeled with the release + /// tag. Restore then fails closed against the pre-upgrade identity + /// instead of waiting for a version the new daemon will never report. + #[test] + fn an_unreadable_release_binary_is_not_labeled_with_the_catalog_tag() { + let catalog = "0.1.0-beta.47"; + let missing = PathBuf::from("/nonexistent/tracedecay-release"); + + let outcome = finish_versioned_upgrade(catalog, Some(missing)); + + let UpgradeOutcome::Installed { version, .. } = outcome else { + panic!("a published release is an install, got {outcome:?}"); + }; + assert_eq!(version, None); + } } // ── Installation ownership ────────────────────────────────────────── From 0d00ec52643413f32daffe9f856d6035cee56ee8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:34:31 +0000 Subject: [PATCH 19/84] fix(hotpath): apply functions limit before live mcp Published hotpath 0.24 snapshots the builder limit for live queries and only reads HOTPATH_FUNCTIONS_LIMIT when the exit report is built. The shipped guard copies that env onto the builder before MCP starts. Co-authored-by: Zack Jackson --- .../using-hotpath/references/hotpath-0.24.md | 2 + .../using-hotpath/references/hotpath-0.24.md | 2 + Cargo.lock | 9 + Cargo.toml | 1 + crates/tracedecay-cli/Cargo.toml | 1 + crates/tracedecay-cli/src/main.rs | 11 +- crates/tracedecay-hotpath-guard/Cargo.toml | 29 ++ crates/tracedecay-hotpath-guard/src/lib.rs | 29 ++ .../tests/functions_limit_live.rs | 344 ++++++++++++++++++ 9 files changed, 425 insertions(+), 3 deletions(-) create mode 100644 crates/tracedecay-hotpath-guard/Cargo.toml create mode 100644 crates/tracedecay-hotpath-guard/src/lib.rs create mode 100644 crates/tracedecay-hotpath-guard/tests/functions_limit_live.rs diff --git a/.claude/skills/using-hotpath/references/hotpath-0.24.md b/.claude/skills/using-hotpath/references/hotpath-0.24.md index a7cb31195e..c4f57b7e57 100644 --- a/.claude/skills/using-hotpath/references/hotpath-0.24.md +++ b/.claude/skills/using-hotpath/references/hotpath-0.24.md @@ -161,6 +161,8 @@ val_logs {"debug_id": 3} Hotpath 0.24 detail calls accept IDs, not names, and have no per-call `limit`. Retention is controlled globally by `HOTPATH_LOGS_LIMIT`. +Published hotpath 0.24 applies `HOTPATH_FUNCTIONS_LIMIT`, else `HOTPATH_LIMIT`, when the exit report is built. Live `functions_timing` and `functions_alloc` use the builder limit captured at guard start. The shipped `tracedecay` process copies that environment onto the builder before the server starts, so a limit set for the process is what those tools return. Setting the variable after the process is already running does not resize the worker. + Recommended order: 1. `profiler_status`. diff --git a/.codex/skills/using-hotpath/references/hotpath-0.24.md b/.codex/skills/using-hotpath/references/hotpath-0.24.md index a7cb31195e..c4f57b7e57 100644 --- a/.codex/skills/using-hotpath/references/hotpath-0.24.md +++ b/.codex/skills/using-hotpath/references/hotpath-0.24.md @@ -161,6 +161,8 @@ val_logs {"debug_id": 3} Hotpath 0.24 detail calls accept IDs, not names, and have no per-call `limit`. Retention is controlled globally by `HOTPATH_LOGS_LIMIT`. +Published hotpath 0.24 applies `HOTPATH_FUNCTIONS_LIMIT`, else `HOTPATH_LIMIT`, when the exit report is built. Live `functions_timing` and `functions_alloc` use the builder limit captured at guard start. The shipped `tracedecay` process copies that environment onto the builder before the server starts, so a limit set for the process is what those tools return. Setting the variable after the process is already running does not resize the worker. + Recommended order: 1. `profiler_status`. diff --git a/Cargo.lock b/Cargo.lock index 3681964872..b6be9ab4b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5642,6 +5642,7 @@ dependencies = [ "tracedecay-global-db", "tracedecay-hooks", "tracedecay-host-integration", + "tracedecay-hotpath-guard", "tracedecay-lcm", "tracedecay-lsp", "tracedecay-maintenance", @@ -6125,6 +6126,14 @@ dependencies = [ "tracedecay-domain", ] +[[package]] +name = "tracedecay-hotpath-guard" +version = "0.1.0" +dependencies = [ + "hotpath", + "serde_json", +] + [[package]] name = "tracedecay-lcm" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 32920d9169..eb65fc59d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ members = [ "crates/tracedecay-host-admission", "crates/tracedecay-host-integration", "crates/tracedecay-hooks", + "crates/tracedecay-hotpath-guard", "crates/tracedecay-lcm", "crates/tracedecay-lsp", "crates/tracedecay-maintenance", diff --git a/crates/tracedecay-cli/Cargo.toml b/crates/tracedecay-cli/Cargo.toml index f3dd06146c..ff3248ad49 100644 --- a/crates/tracedecay-cli/Cargo.toml +++ b/crates/tracedecay-cli/Cargo.toml @@ -179,6 +179,7 @@ tracedecay-dashboard-api = { path = "../tracedecay-dashboard-api", version = "0. tracedecay-domain = { path = "../tracedecay-domain", version = "0.1.0" } tracedecay-global-db = { path = "../tracedecay-global-db", version = "0.1.0" } tracedecay-hooks = { path = "../tracedecay-hooks", version = "0.1.0" } +tracedecay-hotpath-guard = { path = "../tracedecay-hotpath-guard", version = "0.1.0" } tracedecay-host-integration = { path = "../tracedecay-host-integration", version = "0.1.0" } tracedecay-lsp = { path = "../tracedecay-lsp", version = "0.1.0" } tracedecay-maintenance = { path = "../tracedecay-maintenance", version = "0.1.0" } diff --git a/crates/tracedecay-cli/src/main.rs b/crates/tracedecay-cli/src/main.rs index 647c2851fd..3c5a01c7aa 100644 --- a/crates/tracedecay-cli/src/main.rs +++ b/crates/tracedecay-cli/src/main.rs @@ -523,9 +523,14 @@ fn hotpath_guard() -> hotpath::HotpathGuard { // CPU sampling remains available only by explicit operator request: // `HOTPATH_REPORT` (e.g. `functions-cpu`) takes precedence over this // default exclusion. - hotpath::HotpathGuardBuilder::new("tracedecay") - .sections_exclude(vec![hotpath::Section::FunctionsCpu]) - .build() + // Hotpath 0.24 reads HOTPATH_FUNCTIONS_LIMIT only when the exit report is + // built. Live functions_timing and functions_alloc use the builder limit + // captured when this guard starts, so the same env is applied here. + tracedecay_hotpath_guard::with_functions_display_limit( + hotpath::HotpathGuardBuilder::new("tracedecay") + .sections_exclude(vec![hotpath::Section::FunctionsCpu]), + ) + .build() } #[cfg(feature = "hotpath")] diff --git a/crates/tracedecay-hotpath-guard/Cargo.toml b/crates/tracedecay-hotpath-guard/Cargo.toml new file mode 100644 index 0000000000..72f5df1a5b --- /dev/null +++ b/crates/tracedecay-hotpath-guard/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "tracedecay-hotpath-guard" +version = "0.1.0" +publish = false +edition.workspace = true +license = "MIT" +description = "Process-boundary Hotpath display limit for the shipped binary" +repository = "https://github.com/ScriptedAlchemy/tracedecay" + +[lib] +doctest = false + +[features] +# Same activation as the shipped binary: the runtime crate stays crates.io +# hotpath. `hotpath-mcp` is only so the live-query proof can start the server. +hotpath = ["hotpath/hotpath"] +hotpath-mcp = ["hotpath", "hotpath/hotpath-mcp"] + +[dependencies] +hotpath.workspace = true + +[dev-dependencies] +hotpath.workspace = true +serde_json = "1" + +[[test]] +name = "functions_limit_live" +path = "tests/functions_limit_live.rs" +required-features = ["hotpath", "hotpath-mcp"] diff --git a/crates/tracedecay-hotpath-guard/src/lib.rs b/crates/tracedecay-hotpath-guard/src/lib.rs new file mode 100644 index 0000000000..1c09f45fb8 --- /dev/null +++ b/crates/tracedecay-hotpath-guard/src/lib.rs @@ -0,0 +1,29 @@ +//! Process-boundary display cap for the shipped Hotpath guard. +//! +//! Published hotpath 0.24 reads `HOTPATH_FUNCTIONS_LIMIT` (then `HOTPATH_LIMIT`) +//! only when the exit report is built. Live `functions_timing` and +//! `functions_alloc` use the builder limit snapshotted when the guard starts. +//! The shipped binary applies this before that snapshot so a limit already in +//! the process environment is what MCP returns. `0` stays unlimited. A value +//! set after the process is running cannot resize the already started worker. + +/// Applies the functions display limit from the process environment, if set. +/// +/// Unset or unparsable variables leave the builder unchanged, matching the +/// exit report's fallback to the builder default. +pub fn with_functions_display_limit( + builder: hotpath::HotpathGuardBuilder, +) -> hotpath::HotpathGuardBuilder { + match functions_display_limit() { + Some(limit) => builder.functions_limit(limit), + None => builder, + } +} + +fn functions_display_limit() -> Option { + parse_usize_env("HOTPATH_FUNCTIONS_LIMIT").or_else(|| parse_usize_env("HOTPATH_LIMIT")) +} + +fn parse_usize_env(name: &str) -> Option { + std::env::var(name).ok().and_then(|raw| raw.parse().ok()) +} diff --git a/crates/tracedecay-hotpath-guard/tests/functions_limit_live.rs b/crates/tracedecay-hotpath-guard/tests/functions_limit_live.rs new file mode 100644 index 0000000000..b626db2b31 --- /dev/null +++ b/crates/tracedecay-hotpath-guard/tests/functions_limit_live.rs @@ -0,0 +1,344 @@ +//! Live MCP `functions_timing` must honor `HOTPATH_FUNCTIONS_LIMIT` when the +//! shipped guard applies it before the profiler starts. +//! +//! The builder limit is unlimited. Without `with_functions_display_limit`, +//! hotpath 0.24's worker keeps that unlimited snapshot and the tool returns +//! every measured function. The env is set before the guard starts, which is +//! the process boundary the shipped binary has. + +use std::io::{Read, Write}; +use std::net::{SocketAddr, TcpStream}; +use std::time::Duration; + +use tracedecay_hotpath_guard::with_functions_display_limit; + +#[hotpath::measure] +fn slow_long() { + std::thread::sleep(Duration::from_millis(200)); +} + +#[hotpath::measure] +fn slow_mid() { + std::thread::sleep(Duration::from_millis(20)); +} + +#[hotpath::measure] +fn slow_short() { + std::thread::sleep(Duration::from_millis(5)); +} + +#[hotpath::measure] +fn slow_tiny() { + std::thread::sleep(Duration::from_millis(1)); +} + +#[test] +fn live_mcp_functions_timing_honors_functions_limit() { + let port = free_port(); + let report_path = std::env::temp_dir().join(format!( + "hotpath-functions-limit-{}.json", + std::process::id() + )); + let _ = std::fs::remove_file(&report_path); + + unsafe { + std::env::set_var("HOTPATH_EXCLUDE_WRAPPER", "1"); + std::env::set_var("HOTPATH_METRICS_SERVER_OFF", "1"); + std::env::set_var("HOTPATH_MCP_PORT", port.to_string()); + std::env::set_var("HOTPATH_FUNCTIONS_LIMIT", "2"); + std::env::set_var("HOTPATH_OUTPUT_FORMAT", "json"); + std::env::set_var("HOTPATH_OUTPUT_PATH", &report_path); + std::env::remove_var("HOTPATH_LIMIT"); + std::env::remove_var("HOTPATH_REPORT"); + std::env::remove_var("HOTPATH_MCP_AUTH_TOKEN"); + } + + let guard = with_functions_display_limit( + hotpath::HotpathGuardBuilder::new("functions-limit-live") + .functions_limit(0) + .format(hotpath::Format::Json) + .output_path(&report_path), + ) + .build(); + + slow_long(); + slow_mid(); + slow_short(); + slow_tiny(); + + let session = initialize(&port); + let names = wait_for_functions(&port, &session, |got| !got.is_empty()); + assert_live_limit(&names); + + drop(guard); + + let report = std::fs::read_to_string(&report_path).unwrap_or_else(|error| { + panic!("exit report missing at {}: {error}", report_path.display()) + }); + let report: serde_json::Value = serde_json::from_str(&report) + .unwrap_or_else(|error| panic!("exit report is not JSON: {error}\n{report}")); + let exit_names = names_from_list( + report + .get("functions_timing") + .unwrap_or_else(|| panic!("exit report has no functions_timing: {report}")), + ); + assert_live_limit(&exit_names); + let _ = std::fs::remove_file(&report_path); +} + +fn assert_live_limit(names: &[String]) { + assert_eq!( + names.len(), + 2, + "HOTPATH_FUNCTIONS_LIMIT=2 must keep the two slowest functions, got {names:?}" + ); + assert!( + names.iter().any(|name| name.contains("slow_long")), + "missing slow_long in {names:?}" + ); + assert!( + names.iter().any(|name| name.contains("slow_mid")), + "missing slow_mid in {names:?}" + ); + assert!( + names + .iter() + .all(|name| !name.contains("slow_short") && !name.contains("slow_tiny")), + "faster functions leaked past the limit: {names:?}" + ); +} + +fn free_port() -> u16 { + std::net::TcpListener::bind("127.0.0.1:0") + .expect("bind ephemeral port") + .local_addr() + .expect("ephemeral local addr") + .port() +} + +fn initialize(port: &u16) -> String { + let body = r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"functions-limit-live","version":"0"}}}"#; + let mut last = String::new(); + for _ in 0..50 { + match post(port, None, body) { + Ok(response) if response.status == 200 => { + let session = response.header("mcp-session-id").unwrap_or_else(|| { + panic!( + "initialize response missing mcp-session-id: status {} body {}", + response.status, response.body + ) + }); + let notified = post( + port, + Some(session.as_str()), + r#"{"jsonrpc":"2.0","method":"notifications/initialized"}"#, + ) + .unwrap_or_else(|error| panic!("initialized notification failed: {error}")); + assert!( + notified.status == 202 || notified.status == 200, + "initialized notification status {}: {}", + notified.status, + notified.body + ); + return session; + } + Ok(response) => { + last = format!("status {} body {}", response.status, response.body); + } + Err(error) => last = error, + } + std::thread::sleep(Duration::from_millis(20)); + } + panic!("MCP server did not accept initialize on port {port}: {last}"); +} + +fn wait_for_functions(port: &u16, session: &str, ready: impl Fn(&[String]) -> bool) -> Vec { + let mut last = String::new(); + for _ in 0..40 { + match post( + port, + Some(session), + r#"{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"functions_timing","arguments":{}}}"#, + ) { + Ok(response) if response.status == 200 => { + let payload = rpc_result(&response.body); + if payload + .pointer("/result/isError") + .and_then(|value| value.as_bool()) + == Some(true) + { + panic!("functions_timing returned an error: {}", response.body); + } + let text = payload + .pointer("/result/content/0/text") + .and_then(|value| value.as_str()) + .unwrap_or_else(|| { + panic!( + "functions_timing response has no text content: {}", + response.body + ) + }); + let list: serde_json::Value = serde_json::from_str(text).unwrap_or_else(|error| { + panic!("functions_timing text is not JSON: {error}\n{text}") + }); + let names = names_from_list(&list); + if ready(&names) { + return names; + } + last = format!("functions_timing not ready: {names:?}"); + } + Ok(response) => { + last = format!("status {} body {}", response.status, response.body); + } + Err(error) => last = error, + } + std::thread::sleep(Duration::from_millis(25)); + } + panic!("functions_timing did not return the expected measurements: {last}"); +} + +fn names_from_list(list: &serde_json::Value) -> Vec { + list.get("data") + .and_then(|data| data.as_array()) + .unwrap_or_else(|| panic!("function list has no data array: {list}")) + .iter() + .map(|entry| { + entry + .get("name") + .and_then(|name| name.as_str()) + .unwrap_or_else(|| panic!("function entry has no name: {entry}")) + .to_string() + }) + .collect() +} + +fn rpc_result(body: &str) -> serde_json::Value { + let trimmed = body.trim(); + if trimmed.starts_with('{') { + return serde_json::from_str(trimmed) + .unwrap_or_else(|error| panic!("MCP body is not JSON: {error}\n{body}")); + } + let mut found = None; + for line in trimmed.lines() { + let Some(data) = line.trim().strip_prefix("data:") else { + continue; + }; + let data = data.trim(); + if !data.starts_with('{') { + continue; + } + let value: serde_json::Value = serde_json::from_str(data) + .unwrap_or_else(|error| panic!("MCP event is not JSON: {error}\n{data}")); + if value.get("result").is_some() || value.get("error").is_some() { + found = Some(value); + } + } + found.unwrap_or_else(|| panic!("MCP response had no JSON-RPC payload:\n{body}")) +} + +struct HttpResponse { + status: u16, + headers: Vec<(String, String)>, + body: String, +} + +impl HttpResponse { + fn header(&self, name: &str) -> Option { + self.headers.iter().find_map(|(key, value)| { + key.eq_ignore_ascii_case(name) + .then(|| value.trim().to_string()) + }) + } +} + +fn post(port: &u16, session: Option<&str>, body: &str) -> Result { + let address = SocketAddr::from(([127, 0, 0, 1], *port)); + let mut stream = TcpStream::connect_timeout(&address, Duration::from_millis(200)) + .map_err(|error| error.to_string())?; + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .map_err(|error| error.to_string())?; + stream + .set_write_timeout(Some(Duration::from_secs(2))) + .map_err(|error| error.to_string())?; + + let mut request = format!( + "POST /mcp HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nContent-Type: application/json\r\nAccept: application/json, text/event-stream\r\nMCP-Protocol-Version: 2024-11-05\r\nContent-Length: {}\r\nConnection: close\r\n", + body.len() + ); + if let Some(session) = session { + request.push_str(&format!("mcp-session-id: {session}\r\n")); + } + request.push_str("\r\n"); + request.push_str(body); + stream + .write_all(request.as_bytes()) + .map_err(|error| error.to_string())?; + + let mut raw = Vec::new(); + let mut chunk = [0_u8; 4096]; + loop { + match stream.read(&mut chunk) { + Ok(0) => break, + Ok(count) => raw.extend_from_slice(&chunk[..count]), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => break, + Err(error) if error.kind() == std::io::ErrorKind::TimedOut => break, + Err(error) => return Err(error.to_string()), + } + } + parse_http(&raw) +} + +fn parse_http(raw: &[u8]) -> Result { + let text = String::from_utf8_lossy(raw); + let Some((head, body)) = text.split_once("\r\n\r\n") else { + return Err(format!("incomplete HTTP response: {text}")); + }; + let mut lines = head.lines(); + let status_line = lines.next().unwrap_or_default(); + let status = status_line + .split_whitespace() + .nth(1) + .and_then(|code| code.parse::().ok()) + .ok_or_else(|| format!("bad status line: {status_line}"))?; + let headers = lines + .filter_map(|line| { + let (name, value) = line.split_once(':')?; + Some((name.trim().to_string(), value.trim().to_string())) + }) + .collect::>(); + let chunked = headers.iter().any(|(name, value)| { + name.eq_ignore_ascii_case("transfer-encoding") + && value.to_ascii_lowercase().contains("chunked") + }); + let body = if chunked { + decode_chunks(body)? + } else { + body.to_string() + }; + Ok(HttpResponse { + status, + headers, + body, + }) +} + +fn decode_chunks(body: &str) -> Result { + let mut rest = body; + let mut out = String::new(); + loop { + let Some((size_line, after)) = rest.split_once("\r\n") else { + return Err(format!("truncated chunk size: {body}")); + }; + let size = usize::from_str_radix(size_line.trim().split(';').next().unwrap_or(""), 16) + .map_err(|error| format!("bad chunk size {size_line}: {error}"))?; + if size == 0 { + return Ok(out); + } + if after.len() < size { + return Err(format!("truncated chunk of {size} bytes")); + } + out.push_str(&after[..size]); + rest = after.get(size + 2..).unwrap_or(""); + } +} From 88139f6e60392ebc886cbd8fecdf6a00ace89eaa Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:35:06 +0000 Subject: [PATCH 20/84] fix(global-db): classify session projection collisions by field Rebuild activation classified session conflicts as message OutputCollision with message_id session:{id}, erasing the field. Overlaps now use reconcile_session_rows_detailed and SessionOutputCollision. A missing message is no longer reported as a missing session when that session row is still durable. Co-authored-by: Zack Jackson --- .../src/observation_projection.rs | 2 +- .../src/observation_projection/rebuild.rs | 374 ++++++++++++++---- .../src/observation_projection/state.rs | 184 ++++++++- .../src/schema_contract/invariants/audit.rs | 19 +- 4 files changed, 478 insertions(+), 101 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation_projection.rs b/crates/tracedecay-global-db/src/observation_projection.rs index 3b609ffb82..2b58d746cd 100644 --- a/crates/tracedecay-global-db/src/observation_projection.rs +++ b/crates/tracedecay-global-db/src/observation_projection.rs @@ -29,6 +29,6 @@ pub(crate) use state::rearm_queued_projection_retries; #[cfg(test)] pub(super) use state::verify_projection_rows; pub(super) use state::{ - ProjectionOutputAuthority, ProjectionRowsBatch, read_output_authorities, + ProjectionOutputAuthority, ProjectionRowsBatch, load_verified_session, read_output_authorities, read_projection_rows_batch, resolve_output_projection, verify_projection_rows_from_records, }; diff --git a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs index 5f80655c15..57076d0a7c 100644 --- a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs +++ b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs @@ -8,7 +8,7 @@ use tracedecay_lcm::retrieval_content::{ use tracedecay_runtime_core::db::engine::{Connection, TransactionBehavior}; use tracedecay_runtime_core::db::{ Database, - engine::{Executor, QueryExecutor, params}, + engine::{Executor, QueryExecutor, Row, params}, }; use tracedecay_store::{ ObservationProjection, PROVIDER_USAGE_PROJECTOR_VERSION, ProjectedObservation, @@ -2193,76 +2193,128 @@ async fn retire_projection_predecessor_output_ownership( .map_err(|error| storage("retire predecessor projection provenance", error)) } -async fn activate_rebuild_sessions( +fn decode_overlapping_session(row: &Row) -> ProjectionStoreResult { + macro_rules! cell { + ($index:literal) => { + row.get($index) + .map_err(|error| storage("decode overlapping projection session", error))? + }; + ($index:literal, $ty:ty) => { + row.get::<$ty>($index) + .map_err(|error| storage("decode overlapping projection session", error))? + }; + } + Ok(SessionRecord { + provider: cell!(1), + session_id: cell!(2), + project_key: cell!(3), + project_path: cell!(4), + title: cell!(5), + started_at: cell!(6), + ended_at: cell!(7), + transcript_path: cell!(8), + metadata_json: cell!(9), + parent_session_id: cell!(10), + is_subagent: cell!(11, i64) != 0, + agent_id: cell!(12), + parent_tool_use_id: cell!(13), + }) +} + +async fn write_reconciled_session( + conn: &impl Executor, + merged: &SessionRecord, +) -> ProjectionStoreResult<()> { + conn.execute( + "UPDATE sessions + SET project_key = ?3, project_path = ?4, title = ?5, started_at = ?6, + ended_at = ?7, transcript_path = ?8, metadata_json = ?9, + parent_session_id = ?10, is_subagent = ?11, agent_id = ?12, + parent_tool_use_id = ?13 + WHERE provider = ?1 AND session_id = ?2", + params![ + merged.provider.as_str(), + merged.session_id.as_str(), + merged.project_key.as_str(), + merged.project_path.as_str(), + merged.title.as_deref(), + merged.started_at, + merged.ended_at, + merged.transcript_path.as_deref(), + merged.metadata_json.as_deref(), + merged.parent_session_id.as_deref(), + i64::from(merged.is_subagent), + merged.agent_id.as_deref(), + merged.parent_tool_use_id.as_deref(), + ], + ) + .await + .map(|_| ()) + .map_err(|error| storage("activate reconciled projection session", error)) +} + +/// Classify every staged session that already exists through +/// [`reconcile_session_rows_detailed`], the same authority live apply uses. +/// A parallel SQL predicate used to report those conflicts as message +/// `OutputCollision` values with `message_id = session:{id}`, which erased +/// the field and sent session conflicts down the message-skip path. +async fn reconcile_overlapping_rebuild_sessions( conn: &impl Executor, generation: &str, ) -> ProjectionStoreResult<()> { - let mut conflicts = conn + let mut overlaps = conn .query( - "SELECT staged.provider, staged.session_id + "SELECT staged.session_json, + active.provider, active.session_id, active.project_key, + active.project_path, active.title, active.started_at, + active.ended_at, active.transcript_path, active.metadata_json, + active.parent_session_id, active.is_subagent, active.agent_id, + active.parent_tool_use_id FROM observation_projection_rebuild_sessions AS staged JOIN sessions AS active ON active.provider = staged.provider AND active.session_id = staged.session_id - WHERE staged.projector_version = ?1 AND staged.generation = ?2 - AND ( - (active.project_key <> json_extract(staged.session_json, '$.project_key') - AND active.project_key <> 'user' - AND json_extract(staged.session_json, '$.project_key') <> 'user') - OR (active.project_path <> json_extract(staged.session_json, '$.project_path') - AND active.project_path <> active.project_key - AND json_extract(staged.session_json, '$.project_path') - <> json_extract(staged.session_json, '$.project_key')) - OR (active.transcript_path IS NOT NULL - AND json_extract(staged.session_json, '$.transcript_path') IS NOT NULL - AND active.transcript_path IS NOT json_extract(staged.session_json, '$.transcript_path')) - OR (active.parent_session_id IS NOT NULL - AND json_extract(staged.session_json, '$.parent_session_id') IS NOT NULL - AND active.parent_session_id IS NOT json_extract(staged.session_json, '$.parent_session_id')) - OR (active.agent_id IS NOT NULL - AND json_extract(staged.session_json, '$.agent_id') IS NOT NULL - AND active.agent_id IS NOT json_extract(staged.session_json, '$.agent_id')) - OR (active.parent_tool_use_id IS NOT NULL - AND json_extract(staged.session_json, '$.parent_tool_use_id') IS NOT NULL - AND active.parent_tool_use_id IS NOT json_extract(staged.session_json, '$.parent_tool_use_id')) - OR (active.metadata_json IS NOT NULL - AND json_extract(staged.session_json, '$.metadata_json') IS NOT NULL - AND active.metadata_json IS NOT json_extract(staged.session_json, '$.metadata_json') - AND ( - json_valid(active.metadata_json) = 0 - OR json_valid(json_extract(staged.session_json, '$.metadata_json')) = 0 - OR json_type(active.metadata_json) <> 'object' - OR json_type(json_extract(staged.session_json, '$.metadata_json')) <> 'object' - OR EXISTS ( - SELECT 1 - FROM json_each(json_extract(staged.session_json, '$.metadata_json')) AS expected - JOIN json_each(active.metadata_json) AS actual USING (key) - WHERE expected.key NOT IN ('source', 'usage') - AND actual.value IS NOT expected.value - ) - )) - ) - LIMIT 1", + WHERE staged.projector_version = ?1 AND staged.generation = ?2", params![SESSION_MESSAGE_PROJECTOR_VERSION, generation], ) .await - .map_err(|error| storage("validate staged projection sessions", error))?; - if let Some(row) = conflicts + .map_err(|error| storage("read overlapping projection sessions", error))?; + let mut updates = Vec::new(); + while let Some(row) = overlaps .next() .await - .map_err(|error| storage("validate staged projection sessions", error))? + .map_err(|error| storage("read overlapping projection sessions", error))? { - return Err(ProjectionStoreError::OutputCollision { - provider: row - .get(0) - .map_err(|error| storage("validate staged projection sessions", error))?, - message_id: format!( - "session:{}", - row.get::(1) - .map_err(|error| storage("validate staged projection sessions", error))? - ), - }); + let staged_json: String = row + .get(0) + .map_err(|error| storage("read overlapping projection sessions", error))?; + let staged: SessionRecord = decode_json(&staged_json, "decode staged projection session")?; + let actual = decode_overlapping_session(&row)?; + let expected = canonicalize_session_project_paths(&staged); + let normalized_actual = canonicalize_session_project_paths(&actual); + let merged = + reconcile_session_rows_detailed(&normalized_actual, &expected).map_err(|conflict| { + ProjectionStoreError::SessionOutputCollision { + provider: expected.provider.clone(), + session_id: expected.session_id.clone(), + field: conflict.field(), + } + })?; + if merged != actual { + updates.push(merged); + } } - drop(conflicts); + drop(overlaps); + for merged in &updates { + write_reconciled_session(conn, merged).await?; + } + Ok(()) +} + +async fn activate_rebuild_sessions( + conn: &impl Executor, + generation: &str, +) -> ProjectionStoreResult<()> { + reconcile_overlapping_rebuild_sessions(conn, generation).await?; let session_extracts = json_extract_select_list(SESSION_JSON_COLUMN, SESSION_JSON_FIELDS); conn.execute( &format!( @@ -2271,37 +2323,15 @@ async fn activate_rebuild_sessions( transcript_path, metadata_json, parent_session_id, is_subagent, agent_id, parent_tool_use_id ) - SELECT provider, session_id, + SELECT staged.provider, staged.session_id, {session_extracts} - FROM observation_projection_rebuild_sessions - WHERE projector_version = ?1 AND generation = ?2 - ON CONFLICT(provider, session_id) DO UPDATE SET - project_key = CASE - WHEN sessions.project_key = 'user' THEN excluded.project_key - ELSE sessions.project_key END, - project_path = CASE - WHEN sessions.project_path = sessions.project_key THEN excluded.project_path - ELSE sessions.project_path END, - title = COALESCE(sessions.title, excluded.title), - started_at = CASE - WHEN sessions.started_at IS NULL THEN excluded.started_at - WHEN excluded.started_at IS NULL THEN sessions.started_at - ELSE MIN(sessions.started_at, excluded.started_at) END, - ended_at = CASE - WHEN sessions.ended_at IS NULL THEN excluded.ended_at - WHEN excluded.ended_at IS NULL THEN sessions.ended_at - ELSE MAX(sessions.ended_at, excluded.ended_at) END, - transcript_path = COALESCE(sessions.transcript_path, excluded.transcript_path), - metadata_json = CASE - WHEN sessions.metadata_json IS NULL THEN excluded.metadata_json - WHEN excluded.metadata_json IS NULL THEN sessions.metadata_json - ELSE json_patch(excluded.metadata_json, sessions.metadata_json) END, - parent_session_id = COALESCE(sessions.parent_session_id, excluded.parent_session_id), - is_subagent = MAX(sessions.is_subagent, excluded.is_subagent), - agent_id = COALESCE(sessions.agent_id, excluded.agent_id), - parent_tool_use_id = COALESCE( - sessions.parent_tool_use_id, excluded.parent_tool_use_id - )" + FROM observation_projection_rebuild_sessions AS staged + WHERE staged.projector_version = ?1 AND staged.generation = ?2 + AND NOT EXISTS ( + SELECT 1 FROM sessions AS active + WHERE active.provider = staged.provider + AND active.session_id = staged.session_id + )" ), params![SESSION_MESSAGE_PROJECTOR_VERSION, generation], ) @@ -2598,3 +2628,173 @@ async fn activate_rebuild_dispositions( .map(|_| ()) .map_err(|error| storage("activate rebuilt projection dispositions", error)) } + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod activation_tests { + use super::{SESSION_MESSAGE_PROJECTOR_VERSION, activate_rebuild_sessions}; + use crate::tests::harness::RegisteredGlobalDbHarness; + use tracedecay_runtime_core::db::engine::{Executor, params}; + use tracedecay_store::{ProjectionStoreError, SessionRecord}; + + const SESSION_ID: &str = "002bd803-dc62-46e2-b66a-a61cc282f0dc"; + + fn session(transcript_path: Option<&str>, title: Option<&str>) -> SessionRecord { + SessionRecord { + provider: "cursor".to_owned(), + session_id: SESSION_ID.to_owned(), + project_key: "project.fixture".to_owned(), + project_path: "project.fixture".to_owned(), + title: title.map(str::to_owned), + started_at: Some(1), + ended_at: Some(2), + transcript_path: transcript_path.map(str::to_owned), + metadata_json: None, + parent_session_id: None, + is_subagent: false, + agent_id: None, + parent_tool_use_id: None, + } + } + + async fn stage(transaction: &impl Executor, generation: &str, staged: &SessionRecord) { + transaction + .execute( + "INSERT INTO observation_projection_rebuilds ( + projector_version, generation, frontier_sequence, state + ) VALUES (?1, ?2, 0, 'ready')", + params![SESSION_MESSAGE_PROJECTOR_VERSION, generation], + ) + .await + .unwrap(); + transaction + .execute( + "INSERT INTO observation_projection_rebuild_sessions ( + projector_version, generation, provider, session_id, session_json + ) VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + SESSION_MESSAGE_PROJECTOR_VERSION, + generation, + staged.provider.as_str(), + staged.session_id.as_str(), + serde_json::to_string(staged).unwrap().as_str(), + ], + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn activation_names_the_session_field_instead_of_a_message_collision() { + let harness = RegisteredGlobalDbHarness::open("session-collision-field").await; + let active = session(Some("/private/old-transcript.jsonl"), None); + assert!(harness.registered.upsert_session(&active).await); + let transaction = harness.registered.begin_write_transaction().await.unwrap(); + let staged = session(Some("/private/new-transcript.jsonl"), None); + stage(&transaction, "generation.session-collision", &staged).await; + + let error = activate_rebuild_sessions(&transaction, "generation.session-collision") + .await + .expect_err("incompatible transcript paths must not activate"); + match &error { + ProjectionStoreError::SessionOutputCollision { + provider, + session_id, + field, + } => { + assert_eq!(provider, "cursor"); + assert_eq!(session_id, SESSION_ID); + assert_eq!(*field, "transcript_path"); + } + other => panic!("session conflict classified as {other}"), + } + let rendered = error.to_string(); + assert!(rendered.contains("transcript_path")); + assert!(!rendered.contains("session:")); + assert!(!rendered.contains("/private/old-transcript.jsonl")); + assert!(!rendered.contains("/private/new-transcript.jsonl")); + + let mut rows = transaction + .query( + "SELECT transcript_path FROM sessions WHERE provider = 'cursor' AND session_id = ?1", + params![SESSION_ID], + ) + .await + .unwrap(); + let persisted = rows + .next() + .await + .unwrap() + .unwrap() + .get::(0) + .unwrap(); + assert_eq!(persisted, "/private/old-transcript.jsonl"); + } + + #[tokio::test] + async fn activation_merges_a_compatible_session_and_inserts_a_new_one() { + let harness = RegisteredGlobalDbHarness::open("session-collision-merge").await; + let active = session(None, None); + assert!(harness.registered.upsert_session(&active).await); + let transaction = harness.registered.begin_write_transaction().await.unwrap(); + let staged = session(None, Some("Composer session")); + stage(&transaction, "generation.session-merge", &staged).await; + let fresh = SessionRecord { + provider: "cursor".to_owned(), + session_id: "session.fresh".to_owned(), + title: Some("fresh session".to_owned()), + ..active.clone() + }; + transaction + .execute( + "INSERT INTO observation_projection_rebuild_sessions ( + projector_version, generation, provider, session_id, session_json + ) VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + SESSION_MESSAGE_PROJECTOR_VERSION, + "generation.session-merge", + fresh.provider.as_str(), + fresh.session_id.as_str(), + serde_json::to_string(&fresh).unwrap().as_str(), + ], + ) + .await + .unwrap(); + + activate_rebuild_sessions(&transaction, "generation.session-merge") + .await + .unwrap(); + + let mut rows = transaction + .query( + "SELECT title FROM sessions WHERE provider = 'cursor' AND session_id = ?1", + params![SESSION_ID], + ) + .await + .unwrap(); + let title = rows + .next() + .await + .unwrap() + .unwrap() + .get::(0) + .unwrap(); + assert_eq!(title, "Composer session"); + drop(rows); + let mut rows = transaction + .query( + "SELECT title FROM sessions WHERE provider = 'cursor' AND session_id = 'session.fresh'", + (), + ) + .await + .unwrap(); + let fresh_title = rows + .next() + .await + .unwrap() + .unwrap() + .get::(0) + .unwrap(); + assert_eq!(fresh_title, "fresh session"); + } +} diff --git a/crates/tracedecay-global-db/src/observation_projection/state.rs b/crates/tracedecay-global-db/src/observation_projection/state.rs index c3c911e745..2be7d5381d 100644 --- a/crates/tracedecay-global-db/src/observation_projection/state.rs +++ b/crates/tracedecay-global-db/src/observation_projection/state.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::collections::{BTreeSet, HashMap}; use tracedecay_domain::{CanonicalObservationIdV1, DurableObservationV1}; @@ -687,6 +688,26 @@ async fn message_projection( .ok_or(ProjectionStoreError::ProvenanceCollision) } +/// Session row the output verification compares against. +/// +/// The projection-row batch loads sessions from message rows it found. A +/// missing or relocated message therefore has no batch entry even when the +/// expected session row is durable. That absence is not `row_missing`; the +/// single-output path's [`read_session`] is the authority for it. +pub(in super::super) async fn load_verified_session<'a>( + conn: &impl QueryExecutor, + rows: &'a ProjectionRowsBatch, + provider: &str, + session_id: &str, +) -> ProjectionStoreResult>> { + if let Some(session) = rows.session(provider, session_id) { + return Ok(Some(Cow::Borrowed(session))); + } + Ok(read_session(conn, provider, session_id) + .await? + .map(Cow::Owned)) +} + pub(in super::super) async fn verify_projection_rows( conn: &impl QueryExecutor, projection: &SessionMessageProjection, @@ -1396,14 +1417,26 @@ pub(super) async fn protected_message_rows_compatible( #[cfg(test)] #[allow(clippy::unwrap_used)] mod reconcile_tests { - #[cfg(unix)] + use std::collections::BTreeSet; + use crate::tests::harness::RegisteredGlobalDbHarness; + use tracedecay_domain::{ + CanonicalObservationEnvelopeV1, ComponentVersion, ObservationId, + ObservationIdentityMaterialV1, ObservationOrderingDomainV1, ObservationScopeV1, + ObservationSourceGenerationV1, ObservationSourceIdentityV1, ObservationSourceRangeV1, + PayloadReferenceV1, RetentionClass, SanitizationReceiptId, SanitizationReceiptRefV1, + SanitizationReceiptV1, SanitizerDispositionV1, SensitivityV1, + }; #[cfg(unix)] use tracedecay_runtime_core::db::engine::params; - use tracedecay_store::SessionRecord; + use tracedecay_store::{ + ObservationProjection, ProjectionStoreError, SessionMessageRecord, SessionRecord, + }; - use super::canonicalize_session_project_paths; - use super::reconcile_session_rows_detailed; + use super::{ + canonicalize_session_project_paths, load_verified_session, read_projection_rows_batch, + reconcile_session_rows_detailed, verify_projection_rows_from_records, + }; fn record(project_path: &str) -> SessionRecord { SessionRecord { @@ -1608,4 +1641,147 @@ mod reconcile_tests { assert_eq!(conflict.field(), "transcript_path"); } + + #[tokio::test] + async fn missing_message_is_an_output_collision_not_a_missing_session() { + let mut fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../../../tests/fixtures/provider_normalization/codex/agent_message.expected_envelope.json" + )) + .unwrap(); + fixture["stable_record_id"] = + serde_json::Value::String("record.missing-message".to_owned()); + fixture["relations"]["session_id"] = + serde_json::Value::String("session.missing-message".to_owned()); + fixture["relations"]["thread_id"] = + serde_json::Value::String("session.missing-message".to_owned()); + fixture["relations"]["message_id"] = + serde_json::Value::String("record.missing-message".to_owned()); + let envelope: CanonicalObservationEnvelopeV1 = serde_json::from_value(fixture).unwrap(); + let source = ObservationSourceIdentityV1::for_provider( + envelope.provider().clone(), + envelope.relations().session_id().clone(), + ) + .unwrap(); + let payload = serde_json::to_value(&envelope).unwrap(); + let receipt = SanitizationReceiptV1::new( + SanitizationReceiptRefV1::new( + SanitizationReceiptId::new("receipt.missing-message").unwrap(), + ComponentVersion::new("sanitizer.missing-message.v1").unwrap(), + ) + .unwrap(), + SanitizerDispositionV1::Accepted, + SensitivityV1::NonSensitive, + Some(PayloadReferenceV1::for_payload(&payload).unwrap()), + ) + .unwrap(); + let observation = tracedecay_domain::DurableObservationV1::new( + ObservationIdentityMaterialV1::for_native_record( + source, + ObservationScopeV1::Profile, + ObservationSourceGenerationV1::new(1).unwrap(), + ObservationSourceRangeV1::new(0, 100).unwrap(), + ObservationOrderingDomainV1::FileBytes, + ObservationId::new("record.missing-message").unwrap(), + ) + .unwrap(), + receipt, + RetentionClass::new("retention.missing-message").unwrap(), + payload, + ) + .unwrap(); + let session = SessionRecord { + provider: "codex".to_owned(), + session_id: "session.missing-message".to_owned(), + project_key: "user".to_owned(), + project_path: "user".to_owned(), + title: None, + started_at: Some(1), + ended_at: Some(2), + transcript_path: None, + metadata_json: None, + parent_session_id: None, + is_subagent: false, + agent_id: None, + parent_tool_use_id: None, + }; + let message = SessionMessageRecord { + provider: "codex".to_owned(), + message_id: "record.missing-message".to_owned(), + session_id: "session.missing-message".to_owned(), + role: "assistant".to_owned(), + timestamp: Some(1), + ordinal: 0, + text: "The billing pipeline regression is fixed.".to_owned(), + kind: None, + model: None, + tool_names: None, + source_path: None, + source_offset: None, + metadata_json: None, + }; + let projection = ObservationProjection::for_message(&observation, session, message) + .unwrap() + .message() + .expect("explicit message projection") + .clone(); + let message = projection.message(); + let session = projection.session(); + assert_eq!(message.provider, "codex"); + assert_eq!(message.message_id, "record.missing-message"); + assert_eq!(session.session_id, "session.missing-message"); + let outputs = BTreeSet::from([(message.provider.clone(), message.message_id.clone())]); + + let harness = RegisteredGlobalDbHarness::open("missing-message-collision").await; + let absent = harness.registered.read_snapshot().await.unwrap(); + let batch = read_projection_rows_batch(&absent, &outputs).await.unwrap(); + assert!(batch.message("codex", "record.missing-message").is_none()); + assert!( + load_verified_session(&absent, &batch, "codex", "session.missing-message") + .await + .unwrap() + .is_none() + ); + let missing_session = verify_projection_rows_from_records(&absent, &projection, None, None) + .await + .expect_err("a projection with no stored session is a session collision"); + assert!(matches!( + missing_session, + ProjectionStoreError::SessionOutputCollision { + field: "row_missing", + .. + } + )); + + assert!(harness.registered.upsert_session(session).await); + let present = harness.registered.read_snapshot().await.unwrap(); + let batch = read_projection_rows_batch(&present, &outputs) + .await + .unwrap(); + assert!( + batch.session("codex", "session.missing-message").is_none(), + "the message-keyed batch still does not see a session the message row never named" + ); + let loaded = load_verified_session(&present, &batch, "codex", "session.missing-message") + .await + .unwrap() + .expect("the durable session row is not missing"); + let missing_message = verify_projection_rows_from_records( + &present, + &projection, + Some(loaded.as_ref()), + batch.message("codex", "record.missing-message"), + ) + .await + .expect_err("a missing message with a live session is an output collision"); + match missing_message { + ProjectionStoreError::OutputCollision { + provider, + message_id, + } => { + assert_eq!(provider, "codex"); + assert_eq!(message_id, "record.missing-message"); + } + other => panic!("missing message classified as {other}"), + } + } } diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs index 565c0a309d..09b5b64351 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs @@ -816,12 +816,8 @@ async fn validate_message_projection_row( )? == StoredProvenanceRendering::Current { // Convergence supersedes an existing output row; it never inserts one. - // Both repair arms below therefore require the row to be there: a - // vanished output stays the hard failure #1775 and #1781 both promised, - // instead of a recorded repair that writes nothing. The batch also - // derives its session keys from the message rows it found, so a missing - // message is reported as a missing *session* row, which is why this - // guard has to cover the session arm too. + // A vanished message stays a hard failure. Session repair is only the + // uniquely owned current output whose session row is absent. let owner_message = owner_projection.message(); let output_row_present = resolved .projection_rows @@ -877,12 +873,17 @@ async fn verify_owner_output_rows( ) -> std::result::Result<(), ProjectionStoreError> { let session = owner.session(); let message = owner.message(); + let session_row = crate::observation_projection::load_verified_session( + conn, + &resolved.projection_rows, + &session.provider, + &session.session_id, + ) + .await?; crate::observation_projection::verify_projection_rows_from_records( conn, owner, - resolved - .projection_rows - .session(&session.provider, &session.session_id), + session_row.as_deref(), resolved .projection_rows .message(&message.provider, &message.message_id), From 2579de9da1e39d66f28ba4b2ad6415c62c1ecd94 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:35:55 +0000 Subject: [PATCH 21/84] fix(code-index): idle means the pass tail already ran The worker dropped its admission permit and reconcile_in_progress, then still renamed the active pointer and stamped BusyFollowUp. Those two signals are now idle only after that tail: continuations are stamped while the pass is visible, and published text projection re-takes the permit before the pointer rename. The pass is dropped only across that permit wait, so a holder waiting on the flag cannot deadlock. Co-authored-by: Zack Jackson --- .../src/code_index_scheduler/registry.rs | 15 ++ .../code_index_scheduler/registry/mount.rs | 165 +++++++++++++++--- .../src/code_index_scheduler/tests/mod.rs | 13 +- 3 files changed, 156 insertions(+), 37 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry.rs index a5392c83b1..ab44191731 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry.rs @@ -2109,6 +2109,21 @@ impl CodeIndexSchedulerRegistryV1 { } } + /// Stamp a continuation while `reconcile_in_progress` still reports this pass. + /// + /// Callers that already released the worker's pass guard use this so a + /// reader waiting for the counter to hit zero cannot observe an empty + /// slot and then lose to `BusyFollowUp`. The stamp is the idle boundary; + /// the guard lives only for the note. + fn note_visible_worker_continuation( + passes: &Arc, + pending_wake: &PendingWakeV1, + wake: &tokio::sync::Notify, + ) { + let _visible = super::ReconcilePassGuard::enter(passes); + Self::note_worker_continuation(pending_wake, wake); + } + /// Claim the pending wake as one reconcile's arrival, at the instant the /// scheduler dequeues it. /// diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs index ac239ce49c..f488bde488 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs @@ -1031,12 +1031,16 @@ impl CodeIndexSchedulerRegistryV1 { started_micros, ); } - // Source reconciliation is complete: release the background - // admission permit before HeadOpening / graph work so sibling - // stores can start. Keep `reconcile_pass` through text - // seating, dropping it made `reconcile_in_progress` lie while - // this worker still owned graph try_lock, which deadlocked - // tests that hold the scheduler mutex and wait for that flag. + // Source reconciliation is complete. Release the admission + // permit only across HeadOpening's scheduler-mutex wait: a + // holder of that mutex must be able to run, and an + // ignored-dependency owner still needs this permit before it + // can take the mutex. The publication's text projection + // re-acquires the permit before it renames the active pointer. + // Keep `reconcile_pass` through text seating; dropping it + // made `reconcile_in_progress` lie while this worker still + // owned graph try_lock, which deadlocked tests that hold the + // scheduler mutex and wait for that flag. drop(_background_reconcile_admission); // A publication must first reopen its own lightweight text // owner: publication moved the durable pointer, so the prior @@ -1110,6 +1114,49 @@ impl CodeIndexSchedulerRegistryV1 { && !graph_activation_deferred && let Some(text) = graph_text.clone() { + // Head opening released the permit so it could wait on + // the scheduler mutex. Take it back for the pointer + // rename. Drop the pass across that wait: a caller + // holding the permit and waiting for the pass would + // otherwise deadlock, and the pass is re-entered + // before the rename so idle still means the pointer + // write has finished. + let resume_pass = reconcile_pass.is_some(); + drop(reconcile_pass.take()); + let Ok(_text_artifact_admission) = hotpath::future!( + Arc::clone(&worker_background_reconcile_admission).acquire_owned(), + label = "daemon.code_index.admission_wait" + ) + .await + else { + tracing::info!( + event = "code_index_worker_shutdown_observed", + phase = "published_text_projection", + "code-index worker observed shutdown and stopped its pass" + ); + Self::join_retained_text_projection_on_worker_exit( + &mut retained_text_projection, + ) + .await; + return; + }; + if worker_shutting_down.load(Ordering::Acquire) { + tracing::info!( + event = "code_index_worker_shutdown_observed", + phase = "published_text_projection", + "code-index worker observed shutdown and stopped its pass" + ); + Self::join_retained_text_projection_on_worker_exit( + &mut retained_text_projection, + ) + .await; + return; + } + if resume_pass { + reconcile_pass = Some(super::super::ReconcilePassGuard::enter( + &worker_reconcile_in_progress, + )); + } let projection = tokio::spawn(Self::drive_text_projection( text, Arc::clone(&worker_shutting_down), @@ -1168,7 +1215,21 @@ impl CodeIndexSchedulerRegistryV1 { // A successor-only retained projection holds no pass guard of // its own; keeping the worker's guard through graph seat would // report rebuild_in_flight for clone backfill that is not - // exact/lexical work. + // exact/lexical work. Stamp the continuation this projection + // already owes before that drop: the slot, not a later note, + // is what an idle reader observes. + if let Some(outcome) = published_text_projection_outcome.as_ref() { + let schedule_continuation = match outcome { + PublishedTextProjectionOutcomeV1::Finished => graph_text + .as_ref() + .is_some_and(LatestCodeTextGenerationV1::text_projection_needs_work), + PublishedTextProjectionOutcomeV1::Unfinished => true, + PublishedTextProjectionOutcomeV1::Shutdown => false, + }; + if schedule_continuation { + Self::note_worker_continuation(&worker_pending_wake, &worker_wake); + } + } if retained_text_projection.is_none() || retained_projection_successor_only { drop(reconcile_pass.take()); } @@ -1296,6 +1357,14 @@ impl CodeIndexSchedulerRegistryV1 { .filter(|retained| retained.uses_partitioned_manifest()) .cloned() { + // Every outcome of this attempt schedules one successor. + // Stamp it before the recovery await, while the pass is + // visible, so the wait cannot be sampled as an idle slot. + Self::note_visible_worker_continuation( + &worker_reconcile_in_progress, + &worker_pending_wake, + &worker_wake, + ); retained_graph_head_recovery_attempted = true; let generation_id = retained.metadata().manifest().generation_id.clone(); let replay_scheduler = Arc::clone(&worker_scheduler); @@ -1400,8 +1469,8 @@ impl CodeIndexSchedulerRegistryV1 { // all and never published the successor generation. The // `retained_graph_head_recovery_attempted` guard above is // now false for every later pass, so this cannot spin - // another retained-recovery Noop. - Self::note_worker_continuation(&worker_pending_wake, &worker_wake); + // another retained-recovery Noop. The successor was + // stamped before this await. } // A recovered revision-7 verified head already serves its // native graph from the retained text owner, and that owner @@ -1458,6 +1527,8 @@ impl CodeIndexSchedulerRegistryV1 { let graph_text = graph_text.clone(); let shutting_down = Arc::clone(&worker_shutting_down); let prepare_passes = Arc::clone(&worker_reconcile_in_progress); + let prepare_pending_wake = Arc::clone(&worker_pending_wake); + let prepare_wake = Arc::clone(&worker_wake); match hotpath::future!( tokio::task::spawn_blocking(move || { let decoder = Self::lock_scheduler_for_graph_step( @@ -1513,6 +1584,18 @@ impl CodeIndexSchedulerRegistryV1 { )? .1 .take_ignored_roster_refusal_rebuild(); + if roster_refusal_rebuild { + // One pass, claimed from the scheduler, so + // a refusal that keeps reproducing cannot + // spin this worker. Stamp before this + // closure drops the step guard: the result + // is observed only after the slot is set. + Self::note_visible_worker_continuation( + &prepare_passes, + &prepare_pending_wake, + &prepare_wake, + ); + } let replay_binding = match latest.as_ref() { Some(latest) => Some( Self::lock_scheduler_for_graph_step( @@ -1545,15 +1628,6 @@ impl CodeIndexSchedulerRegistryV1 { the sealed generation cannot seat" ); } - if roster_refusal_rebuild { - // One pass, claimed from the scheduler, so - // a refusal that keeps reproducing cannot - // spin this worker. - Self::note_worker_continuation( - &worker_pending_wake, - &worker_wake, - ); - } Ok((outcome, latest, replay_binding)) } Ok(Err(error)) => { @@ -1714,7 +1788,14 @@ impl CodeIndexSchedulerRegistryV1 { .as_ref() .is_some_and(LatestCodeTextGenerationV1::text_projection_needs_work) { - Self::note_worker_continuation(&worker_pending_wake, &worker_wake); + // Already stamped before optional graph. Re-enter + // the pass so a reader that cleared the slot + // during graph still cannot sample the stamp. + Self::note_visible_worker_continuation( + &worker_reconcile_in_progress, + &worker_pending_wake, + &worker_wake, + ); } // Large text projections can outlive the bounded // source proof established before publication. The @@ -1784,7 +1865,11 @@ impl CodeIndexSchedulerRegistryV1 { "the publication's text owner did not finish its projection; \ the sealed generation stays unseated until it does" ); - Self::note_worker_continuation(&worker_pending_wake, &worker_wake); + Self::note_visible_worker_continuation( + &worker_reconcile_in_progress, + &worker_pending_wake, + &worker_wake, + ); } } // Keep the pass lifetime around the post-projection source @@ -1965,7 +2050,11 @@ impl CodeIndexSchedulerRegistryV1 { if text_latest.text_projection_needs_work() && !text_latest.query_owners_are_ready() { - Self::note_worker_continuation(&worker_pending_wake, &worker_wake); + Self::note_visible_worker_continuation( + &worker_reconcile_in_progress, + &worker_pending_wake, + &worker_wake, + ); } } Ok(Err(error)) => { @@ -1994,7 +2083,27 @@ impl CodeIndexSchedulerRegistryV1 { } // The source proof and serving witness are now published as // one lifecycle. Optional receipts do not keep source - // verification in flight. + // verification in flight. A clone-backfill continuation this + // pass already knows about is stamped first, so the drop is + // not an empty slot. + if clone_backfill_waiting_for_source + && matches!( + &result, + Ok((Ok(CodeIndexReconcileOutcomeV1::Noop(_)), _, _)) + ) + && worker_text_generation + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .is_some() + && worker_source_freshness + .ready_without_stat(&worker_project_root, &worker_shutting_down) + { + Self::note_visible_worker_continuation( + &worker_reconcile_in_progress, + &worker_pending_wake, + &worker_wake, + ); + } drop(reconcile_pass.take()); if let Ok((Ok(outcome), _, _)) = &result { // A pass that ran to a terminal outcome proves neither the @@ -2046,12 +2155,8 @@ impl CodeIndexSchedulerRegistryV1 { ); } worker_serving_generation_changed.send_replace(()); - // The retained slice was checked before reconciliation - // renewed this proof. Preserve its wake now that source - // is current, without requiring another query arrival. - if clone_backfill_waiting_for_source { - Self::note_worker_continuation(&worker_pending_wake, &worker_wake); - } + // The clone-backfill continuation was stamped before + // this pass dropped `reconcile_in_progress`. } } else { // Surface bounded non-terminal failure without new project-path data. @@ -2269,7 +2374,6 @@ impl CodeIndexSchedulerRegistryV1 { PublishedTextProjectionOutcomeV1::Unfinished } }; - drop(reconcile_pass.take()); match outcome { PublishedTextProjectionOutcomeV1::Finished if !retained_head_recovered_without_complete_replay @@ -2316,6 +2420,9 @@ impl CodeIndexSchedulerRegistryV1 { Self::note_worker_continuation(&worker_pending_wake, &worker_wake); } } + // The continuation is already in the slot. Dropping here + // is the first moment this pass looks idle. + drop(reconcile_pass.take()); } if worker_shutting_down.load(Ordering::Acquire) { tracing::info!( diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs index c82155ff9b..8cf50ea13c 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs @@ -1338,14 +1338,11 @@ async fn settled_owner_with_idle_admission( /// done disturbing it. /// /// The caller must already hold the single background admission, so no further -/// pass can start. One pass can still be finishing: the worker releases that -/// admission halfway through its body and drops its `reconcile_pass` guard -/// before the branches that call `note_worker_continuation`, so both -/// `reconcile_in_progress` and the slot read quiet while the tail is still -/// about to stamp `BusyFollowUp` into it. [`wait_for_settled_owner`] samples -/// exactly those two, so it cannot see that tail. With the admission held the -/// tail is finite and unrepeatable, so clearing until the slot survives a quiet -/// window is the proof the settle cannot give. +/// pass can start. A pass stamps `BusyFollowUp` before it drops +/// `reconcile_in_progress`, but a notify already banked by that pass can still +/// be claimed the moment the permit is released. Clearing until the slot +/// survives a quiet window is the proof the settle cannot give once that +/// release is the next thing that happens. async fn clear_pending_wake_until_quiet( registry: &CodeIndexSchedulerRegistryV1, scope: &tracedecay_contracts::ResolvedScope, From 91c4bf9653553e797d0807592cb6aa3810b3e778 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:36:04 +0000 Subject: [PATCH 22/84] fix(code-index): keep clone copy off the freshness receipt The advance that installs exact and lexical owners also copied the prior lexical artifact into the clone successor. That copy ran under reconcile_in_progress, so status stayed non-current for the copy. Leave the successor pending; the retained driver starts it after the seat. Co-authored-by: Zack Jackson --- .../src/code_index_scheduler/serving.rs | 17 ++-- .../src/code_index_scheduler/tests/serving.rs | 89 +++++++++++++++++++ 2 files changed, 100 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs index 394cc15419..bf72387e38 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs @@ -3273,7 +3273,7 @@ impl LatestCodeTextGenerationV1 { )); }; drop(slot); - let mut publish_claim = TextHeadOpenClaimV1::new(&self.text_projection_build); + let _publish_claim = TextHeadOpenClaimV1::new(&self.text_projection_build); let CodeTextArtifactBuildV1 { builder, source, @@ -3312,7 +3312,6 @@ impl LatestCodeTextGenerationV1 { ) .map_err(map_text_artifact_error)?; let needs_clone_successor = !reader.has_clone_fingerprints(); - let prior = reader.verified_artifact().clone(); // Match the cold-open path: install owners first, then publish Ready. // Publishing Ready before a failed install (admission ceiling / shrink) // would leave dashboard/MCP progress claiming a ready generation that @@ -3320,10 +3319,16 @@ impl LatestCodeTextGenerationV1 { self.install_artifact_owners(reader, reader_reservation)?; self.publish_text_progress_phase(CodeIndexBuildPhaseV1::Ready, 0, 0); if needs_clone_successor { - let source = store.open_sealed_source(&sealed_identity, control)?; - let build = - self.begin_clone_successor(descriptor, prior, sealed_identity, source, control)?; - drop(publish_claim.install(TextHeadOpenBuildV1::CloneSuccessor(build))); + // `begin_clone_successor` copies the whole prior lexical artifact + // before the first page walk. Doing that here kept this advance, + // and the publication pass awaiting it, inside `reconcile_in_progress` + // for the copy. Exact and lexical serving are already installed; + // the copy is not a freshness precondition. Leave the slot pending + // so the retained driver starts the successor after the seat, + // without the receipt guard. The claim stays armed: its drop + // restores only `HeadOpening`, so `CloneSuccessorPending` survives + // and parked wakes are notified. + self.text_projection_build.retain_clone_successor_retry()?; return Ok(false); } Ok(true) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs index c6b185ada1..f8bf385de0 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs @@ -853,6 +853,95 @@ fn clone_successor_keeps_lexical_owners_ready_and_cas_replaces_v14() { assert_eq!(v16_revision, 16); } +/// Exact and lexical readiness is not the clone-successor copy. +/// +/// The publication advance that installs those owners used to call +/// `begin_clone_successor` before returning, and that call copies the whole +/// prior lexical artifact. The freshness receipt awaits that advance, so +/// status stayed non-current for the copy. The successor must still be +/// reported as backfill, and the next advance is what writes its staging file. +#[test] +fn lexical_readiness_leaves_the_clone_successor_uncopied() { + let fixture = GitFixture::new(&[( + "src/lib.rs", + "pub fn alpha() { one(); two(); three(); four(); five(); six(); seven(); eight(); nine(); ten(); }\n", + )]); + let store = TempDir::new().expect("store root"); + let mut scheduler = scheduler( + &fixture, + store.path().to_path_buf(), + Arc::new(SharedCodeIndexBytePoolV1::default()), + ); + published(scheduler.reconcile_now().expect("publish generation")); + let latest = scheduler.latest_complete().expect("latest generation"); + while !latest.query_owners_are_ready() { + latest.advance_text_serving(1).expect("advance V14 build"); + } + let tracedecay_contracts::code_index_freshness::CodeCloneIndexStatusV1::Backfilling { + observation, + } = latest.clone_index_status(false, None) + else { + panic!( + "a generation without clone fingerprints must report backfill once lexical owners serve, got {:?}", + latest.clone_index_status(false, None) + ); + }; + assert_eq!(observation.coverage.completed_source_pages, 0); + assert!( + observation.coverage.total_source_pages > 0, + "the pending successor must name the sealed page count it has not visited" + ); + // Status falls back to the published artifact's bytes when the successor + // has not created a staging file, so the bytes field cannot prove the + // copy stayed off this advance. The slot and the artifacts directory can. + assert!( + matches!( + &*latest.text_projection_build.lock_slot(), + super::super::CodeTextProjectionSlotV1::CloneSuccessorPending + ), + "owner readiness must leave the successor pending" + ); + let staging_names = |root: &std::path::Path| { + std::fs::read_dir(code_text_artifacts_root(root)) + .expect("artifacts root") + .map(|entry| entry.expect("artifact entry").file_name()) + .filter(|name| name.to_string_lossy().ends_with(".staging")) + .collect::>() + }; + assert!( + staging_names(store.path()).is_empty(), + "owner readiness copied the prior lexical artifact: {:?}", + staging_names(store.path()) + ); + + latest + .advance_text_serving(1) + .expect("the retained successor advance copies the prior artifact"); + assert!(latest.query_owners_are_ready()); + assert!( + !matches!( + &*latest.text_projection_build.lock_slot(), + super::super::CodeTextProjectionSlotV1::CloneSuccessorPending + ), + "the next advance must take the pending successor" + ); + + while latest.text_projection_needs_work() { + latest + .advance_text_serving(16) + .expect("finish clone successor"); + } + let revision: i64 = rusqlite::Connection::open(active_text_artifact_path(store.path())) + .expect("open finished artifact") + .query_row( + "SELECT format_revision FROM artifact_state WHERE singleton = 1", + [], + |row| row.get(0), + ) + .expect("read finished revision"); + assert_eq!(revision, 16); +} + #[test] fn clone_status_distinguishes_unavailable_backfill_partial_ready_and_stale() { let fixture = GitFixture::new(&[( From 8c9243400de6640680f440a1e5532de786e40d6c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:37:18 +0000 Subject: [PATCH 23/84] fix(search-eval): pin ranking, not generation identity Extractor revision bumps reseal the generation named inside every candidate occurrence and the eval freshness digest. Hashing that production fallback subpayload made ranking-identical runs rewrite both partition receipts and the workload identity. The receipt now hashes ordered ranking rows and lane coverage. Workload identity omits those receipts, so a receipt edit is not a workload change. Co-authored-by: Zack Jackson --- .../query-lexical-graph-workload-v1.json | 4 +- .../src/search_quality/candidate_output.rs | 32 ++- .../src/search_quality/packaged.rs | 24 +- .../src/bin/tracedecay-search-eval-direct.rs | 5 +- .../src/candidate_output.rs | 205 +++++++++++++++--- crates/tracedecay-search-eval/src/lib.rs | 5 +- .../src/report_tests.rs | 14 +- 7 files changed, 236 insertions(+), 53 deletions(-) diff --git a/crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json b/crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json index 4646d75e25..1079dcf001 100644 --- a/crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json +++ b/crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json @@ -135,8 +135,8 @@ } ], "expected_query_fallback_digests": { - "train": "sha256:5750d4a588f7a7e14c381ec3a4285400a29e164babbf88e677ce7441aaf8e9b2", - "validation": "sha256:e7a459efb1655bb71e30fd302690cd5ac7937197add94a3ac1cab5812a1f5a48" + "train": "sha256:07c58382a8c5589236e2d29c91827825780dfc1aa11123bccd0f230805671e77", + "validation": "sha256:72b3500daa214348c143e25dab5f367c8bd723035e0471a48526bf3f55c9b54b" }, "profile_matrix": [ { diff --git a/crates/tracedecay-query/src/search_quality/candidate_output.rs b/crates/tracedecay-query/src/search_quality/candidate_output.rs index bd62d10937..e329d77b26 100644 --- a/crates/tracedecay-query/src/search_quality/candidate_output.rs +++ b/crates/tracedecay-query/src/search_quality/candidate_output.rs @@ -72,6 +72,13 @@ pub struct CandidateWorkloadV1 { pub execution_contract: EvaluationExecutionContractV1, pub corpus: Vec, pub profile_matrix: Vec, + /// Observed ranking receipts for `train` and `validation`. + /// + /// These bind ordered ranking rows and public lane coverage. They are not + /// workload inputs: [`compute_workload_digest`] omits them, so re-pinning a + /// receipt does not rewrite the packaged workload identity. Generation and + /// extractor-revision changes reseal candidate occurrence ids without + /// changing those rows, and must not move this field either. pub expected_query_fallback_digests: BTreeMap, pub queries: Vec, } @@ -393,7 +400,12 @@ pub fn load_candidate_workload(path: &Path) -> Result Result { - canonical_sha256(workload) + // Ranking receipts observe the run. Including them made a receipt re-pin + // look like a different workload, including when only a sealed generation + // id moved. + let mut identity = workload.clone(); + identity.expected_query_fallback_digests.clear(); + canonical_sha256(&("tracedecay.search-eval.workload-identity.v1", &identity)) } pub fn compute_profile_material_digest( @@ -1014,6 +1026,24 @@ mod need_provenance_tests { ); } + #[test] + fn ranking_receipt_edits_do_not_move_workload_identity() { + let workload = workload(); + let identity = super::compute_workload_digest(&workload).expect("workload identity"); + assert_eq!(identity, packaged::WORKLOAD_SHA256); + let mut moved = workload; + let train = moved + .expected_query_fallback_digests + .get_mut("train") + .expect("train receipt"); + *train = format!("sha256:{}", "ab".repeat(32)); + assert_eq!( + super::compute_workload_digest(&moved).expect("moved identity"), + identity, + "re-pinning a ranking receipt must not rewrite the workload identity" + ); + } + #[test] fn a_need_without_documented_provenance_is_refused() { let mut workload = workload(); diff --git a/crates/tracedecay-query/src/search_quality/packaged.rs b/crates/tracedecay-query/src/search_quality/packaged.rs index 4b3556a777..1417d88239 100644 --- a/crates/tracedecay-query/src/search_quality/packaged.rs +++ b/crates/tracedecay-query/src/search_quality/packaged.rs @@ -1,13 +1,19 @@ -use tracedecay_domain::canonical_text::sha256_hex; - use super::candidate_output::{ - CandidateWorkloadV1, validate_need_provenance_against_embedded_corpus, + CandidateWorkloadV1, compute_workload_digest, validate_need_provenance_against_embedded_corpus, validate_workload_for_tuning, }; use super::evaluate::SearchEvalError; const WORKLOAD_PATH: &str = "tests/fixtures/search_quality/query-lexical-graph-workload-v1.json"; -const WORKLOAD_SHA256: &str = "267e2bd2e9b90d258cbeed829920ab735f6af0ebc2a6e870d59eeef29b1cdb93"; +/// Canonical identity of the packaged workload inputs. +/// +/// This is [`compute_workload_digest`]: schema, queries, corpus, profile, and +/// execution contract. It deliberately excludes `expected_query_fallback_digests`. +/// Those receipts observe ranking; folding them into this pin made every +/// receipt edit, including a generation-only reseal, rewrite the workload +/// identity as well. +pub const WORKLOAD_SHA256: &str = + "sha256:883b1dc8673f0bdf09f54fd4e4bc598e7df01607933a6bf4053f31003b111d31"; const FILES: &[(&str, &[u8])] = &[ ( @@ -110,15 +116,17 @@ pub fn packaged_evaluator_files() -> &'static [(&'static str, &'static [u8])] { #[hotpath::measure(label = "search_eval.packaged.load_workload")] pub fn load_workload() -> Result { - let observed_workload_digest = sha256_hex(FILES[0].1); + let workload = serde_json::from_slice::(FILES[0].1).map_err(|error| { + SearchEvalError::Contract(format!("parse packaged evaluator workload: {error}")) + })?; + let observed_workload_digest = compute_workload_digest(&workload).map_err(|error| { + SearchEvalError::Contract(format!("hash packaged evaluator workload: {error}")) + })?; if observed_workload_digest != WORKLOAD_SHA256 { return Err(SearchEvalError::Contract(format!( "packaged evaluator workload digest mismatch: expected {WORKLOAD_SHA256}, observed {observed_workload_digest}" ))); } - let workload = serde_json::from_slice::(FILES[0].1).map_err(|error| { - SearchEvalError::Contract(format!("parse packaged evaluator workload: {error}")) - })?; validate_workload_for_tuning(&workload)?; validate_need_provenance_against_embedded_corpus(&workload, FILES)?; Ok(workload) diff --git a/crates/tracedecay-search-eval/src/bin/tracedecay-search-eval-direct.rs b/crates/tracedecay-search-eval/src/bin/tracedecay-search-eval-direct.rs index eca9235503..b353907380 100644 --- a/crates/tracedecay-search-eval/src/bin/tracedecay-search-eval-direct.rs +++ b/crates/tracedecay-search-eval/src/bin/tracedecay-search-eval-direct.rs @@ -239,14 +239,15 @@ mod tests { } #[test] - fn default_validation_uses_the_byte_pinned_packaged_workload() { + fn default_validation_binds_the_packaged_workload_identity() { let summary = validate_requested_workload(std::path::Path::new("."), None) .expect("packaged workload validates"); assert_eq!(summary.status, DirectEvaluationStatusV1::Pass); assert_eq!( summary.workload_digest, - "sha256:8657aa486a4c58e17c9969c7aa5d143a4d30e88dca7d26f13e61c7d3effab091" + tracedecay_query::search_quality::packaged::WORKLOAD_SHA256, + "validate must return the packaged workload identity, not a second pin" ); assert_eq!(summary.profile_count, 1); assert_eq!(summary.query_count, 67); diff --git a/crates/tracedecay-search-eval/src/candidate_output.rs b/crates/tracedecay-search-eval/src/candidate_output.rs index 35167dbfcd..43d6772e12 100644 --- a/crates/tracedecay-search-eval/src/candidate_output.rs +++ b/crates/tracedecay-search-eval/src/candidate_output.rs @@ -5,7 +5,7 @@ //! lexical, and graph production lanes. //! //! Outputs deterministic checked-in `train` / `validation` candidate records -//! plus current/10x resource samples and fallback digests. Cancellation is +//! plus current/10x resource samples and ranking receipts. Cancellation is //! proved fail-closed before those records are returned; it is not restamped //! as a policy field. Labels are ordinary reviewable fixture data, never a //! production authority. @@ -44,12 +44,11 @@ use tracedecay_domain::{ EphemeralSanitizedQueryViewV1, ExactAdmissionRuleRevision, ExactClass, FileOccurrenceId, LanguageId, ManifestDigest, PolicyRevisionId, PrincipalId, PrivacyDomainId, ProjectId, ProjectionBatchRequestV1, ProjectionKeyV1, ProjectionKindV1, ProjectionOperationV1, - ProjectionOutcomeV1, PublicRetrieverStatus, QueryFallbackSubpayload, - QueryNormalizationRevision, RelationEdgeKindV1, RepositoryDirtyStateV1, RepositoryId, - RetrievalFailure, RetrievalRequest, RetrievalScope, RetrievalSnapshot, RetrieverKind, - RetrieverOutcome, SanitizationReceiptId, SanitizedCodeFileV1, SanitizedCodeSnapshotV1, - SanitizerRevision, SingleRootScopeV1, SnapshotFileDispositionV1, SymbolOccurrenceId, - TemporalModeV1, UtcMicros, VectorWatermark, + ProjectionOutcomeV1, PublicRetrieverStatus, QueryNormalizationRevision, RelationEdgeKindV1, + RepositoryDirtyStateV1, RepositoryId, RetrievalFailure, RetrievalRequest, RetrievalScope, + RetrievalSnapshot, RetrieverKind, RetrieverOutcome, SanitizationReceiptId, SanitizedCodeFileV1, + SanitizedCodeSnapshotV1, SanitizerRevision, SingleRootScopeV1, SnapshotFileDispositionV1, + SymbolOccurrenceId, TemporalModeV1, UtcMicros, VectorWatermark, }; use tracedecay_query::native_git::NativeHistoricalBlobReaderV1; use tracedecay_query::retrieval::exact::{ @@ -593,10 +592,20 @@ fn generate_partition_output( let peak_before = peak_rss_bytes(); for query in &queries { let started = Instant::now(); - // The row and both partition fallback digests share one composition. + // The row and both partition receipts share one composition. The + // receipt is the ordered ranking, not the generation-scoped fallback + // subpayload: that digest moves whenever extractor revisions reseal + // the generation every occurrence id names. let composed = compose_production_query(published, profile, query)?; - let fallback = query_fallback_from_composition(&composed)?; - fallback_digests.push((query.query_id.as_str(), fallback.digest.as_str().to_owned())); + let ranked = map_ranked_candidates(published, &composed)?; + let coverage = query_lane_coverage(&composed); + let receipt = ranking_receipt_digest( + composed.profile_id.as_str(), + query.query_id.as_str(), + &coverage, + &ranked, + )?; + fallback_digests.push((query.query_id.as_str(), receipt)); rows.push(query_row_from_composition(published, query, &composed)?); latencies_us.push(started.elapsed().as_micros() as u64); } @@ -609,7 +618,7 @@ fn generate_partition_output( ); let fallback_digest = canonical_sha256(&( - "tracedecay.search-eval.partition-fallbacks.v1", + "tracedecay.search-eval.partition-rankings.v1", &fallback_digests, ))?; let query_digest = fallback_digest.clone(); @@ -839,9 +848,9 @@ fn compose_production_query( .map_err(|error| CandidateOutputError::Contract(error.to_string())) } -fn query_fallback_from_composition( +fn query_lane_coverage( output: &CompositionOutputV1, -) -> Result { +) -> BTreeMap { let mut coverage = BTreeMap::new(); for lane in RetrieverKind::QUERY_FALLBACK_LANES { coverage.insert( @@ -853,41 +862,50 @@ fn query_fallback_from_composition( .unwrap_or(PublicRetrieverStatus::Unavailable), ); } - let fallback = QueryFallbackSubpayload::new( - output.profile_id.clone(), - output.ranked_candidates.clone(), - coverage, - output.freshness.clone(), - None, - ) - .map_err(|error| CandidateOutputError::Contract(error.to_string()))?; - fallback - .validate() - .map_err(|error| CandidateOutputError::Contract(error.to_string()))?; - Ok(fallback) + coverage +} + +/// Ranking identity the search-eval pins compare. +/// +/// Production `QueryFallbackSubpayload` digests stay generation-scoped: every +/// lexical occurrence id is `code-chunk:{generation}:{chunk}`, and the eval +/// request's freshness digest names that generation, whose fingerprint includes +/// extractor revisions. Hashing that subpayload made an extractor revision +/// bump look like a ranking change. This receipt hashes the generation-free +/// rows the quality report already scores, plus the public lane coverage. +fn ranking_receipt_digest( + profile_id: &str, + query_id: &str, + lane_coverage: &BTreeMap, + ranked: &[RankedCandidateRowV1], +) -> Result { + canonical_sha256(&( + "tracedecay.search-eval.ranking-receipt.v1", + profile_id, + query_id, + lane_coverage, + ranked, + )) } fn map_ranked_candidates( published: &PublishedCorpus, output: &CompositionOutputV1, ) -> Result, CandidateOutputError> { - map_ranked_candidate_list(published, &output.ranked_candidates) + map_ranked_candidate_list(&published.occurrence_map, &output.ranked_candidates) } fn map_ranked_candidate_list( - published: &PublishedCorpus, + occurrence_map: &BTreeMap, ranked_candidates: &[tracedecay_domain::RankedCandidate], ) -> Result, CandidateOutputError> { let mut rows = Vec::new(); for ranked in ranked_candidates { - let entry = published - .occurrence_map + let entry = occurrence_map .get(ranked.candidate.anchor_id.as_str()) .or_else(|| { ranked.candidate.occurrences.iter().find_map(|occurrence| { - published - .occurrence_map - .get(occurrence.source_occurrence_id.as_str()) + occurrence_map.get(occurrence.source_occurrence_id.as_str()) }) }) .cloned() @@ -1541,6 +1559,129 @@ pub(crate) mod tests { packaged_fixture().workload().clone() } + fn ranked_with_generation(generation: &str) -> tracedecay_domain::RankedCandidate { + let occurrence = tracedecay_domain::OccurrenceProvenance { + source_occurrence_id: id(&format!("code-chunk:{generation}:chunk.stable")) + .expect("occurrence id"), + file_occurrence_id: Some(id("file.time").expect("file id")), + retriever_evidence_anchor: tracedecay_domain::RetrievalAnchorId::new("evidence.stable") + .expect("evidence anchor"), + source_namespace: id("ns.code.daemon").expect("namespace"), + repository_id: None, + session_or_thread_id: None, + logical_copy_cluster_id: None, + logical_copy_evidence_anchor: None, + evidence_role: tracedecay_domain::EvidenceRole::Primary, + freshness: tracedecay_domain::SourceFreshness { + source_namespace: id("ns.code.daemon").expect("freshness namespace"), + source_instance: id("instance.code-index.daemon").expect("instance"), + source_watermark: Some(1), + projection_watermark: Some(1), + observed_at: UtcMicros(1_000_000), + source_generation: Some(1), + generation_lag: Some(0), + compatibility: tracedecay_domain::FreshnessCompatibilityV1::Current, + policy_revision: id("policy.candidate.v1").expect("policy"), + }, + }; + tracedecay_domain::RankedCandidate { + candidate: tracedecay_domain::FusedCandidate { + anchor_id: tracedecay_domain::RetrievalAnchorId::new("code-symbol:symbol.stable") + .expect("anchor"), + logical_evidence_id: id("code-symbol:symbol.stable").expect("evidence"), + occurrences: vec![occurrence], + exact_class: ExactClass::Approximate, + utility_micros: 1, + contributions: Vec::new(), + freshness: Vec::new(), + decisions: Vec::new(), + }, + final_ordinal: 0, + } + } + + /// Extractor revision bumps reseal the generation embedded in every + /// `code-chunk:{generation}:{chunk}` occurrence id. The production + /// fallback digest binds that id; the ranking receipt must not. + #[test] + fn ranking_receipt_ignores_generation_scoped_occurrence_ids() { + let generation_a = "generation.v1.aaaaaaaa.00000001"; + let generation_b = "generation.v1.bbbbbbbb.00000002"; + let mut map = BTreeMap::new(); + map.insert( + "code-symbol:symbol.stable".to_owned(), + OccurrenceMapEntry { + document_id: "time".to_owned(), + scope: "research".to_owned(), + display_anchors: vec!["time::UtcMicros".to_owned()], + }, + ); + let rows_a = map_ranked_candidate_list(&map, &[ranked_with_generation(generation_a)]) + .expect("map generation a"); + let rows_b = map_ranked_candidate_list(&map, &[ranked_with_generation(generation_b)]) + .expect("map generation b"); + assert_eq!( + rows_a, rows_b, + "display rows are keyed by the generation-free anchor" + ); + + let coverage = BTreeMap::from([ + (RetrieverKind::ExactLiteral, PublicRetrieverStatus::Complete), + (RetrieverKind::Lexical, PublicRetrieverStatus::Complete), + (RetrieverKind::Graph, PublicRetrieverStatus::Unavailable), + ]); + let receipt = + |rows: &[RankedCandidateRowV1], + lanes: &BTreeMap| { + ranking_receipt_digest("profile.query-fallback", "train-001", lanes, rows) + .expect("ranking receipt") + }; + let receipt_a = receipt(&rows_a, &coverage); + assert_eq!(receipt_a, receipt(&rows_b, &coverage)); + + let mut coverage_changed = coverage.clone(); + coverage_changed.insert(RetrieverKind::Graph, PublicRetrieverStatus::Complete); + assert_ne!( + receipt_a, + receipt(&rows_a, &coverage_changed), + "lane coverage is part of the ranking receipt" + ); + let mut reordered = rows_a; + reordered.push(RankedCandidateRowV1 { + anchor: "code-chunk:chunk.other".to_owned(), + anchors: vec!["watermark::merge_max".to_owned()], + scope: "research".to_owned(), + document_id: "watermark".to_owned(), + tier: "approximate".to_owned(), + }); + assert_ne!( + receipt_a, + receipt(&reordered, &coverage), + "a different ranked set must move the receipt" + ); + + let production_digest = |generation: &str| { + let lanes = RetrieverKind::QUERY_FALLBACK_LANES + .into_iter() + .map(|lane| (lane, PublicRetrieverStatus::Complete)) + .collect(); + tracedecay_domain::QueryFallbackSubpayload::new( + id("profile.query-fallback").expect("profile"), + vec![ranked_with_generation(generation)], + lanes, + Vec::new(), + None, + ) + .expect("production fallback subpayload") + .digest + }; + assert_ne!( + production_digest(generation_a).as_str(), + production_digest(generation_b).as_str(), + "the production fallback digest still moves with the sealed generation" + ); + } + #[test] fn fusion_profile_carries_the_checked_in_lane_weights() { let workload = workload(); diff --git a/crates/tracedecay-search-eval/src/lib.rs b/crates/tracedecay-search-eval/src/lib.rs index 27e91ed35c..41091a12fd 100644 --- a/crates/tracedecay-search-eval/src/lib.rs +++ b/crates/tracedecay-search-eval/src/lib.rs @@ -54,10 +54,11 @@ pub fn default_workload_path(repo_root: &Path) -> PathBuf { repo_root.join(WORKLOAD_RELATIVE) } -/// Validate the byte-pinned packaged workload. +/// Validate the packaged workload identity. /// /// Ordinary developer comparisons may use an explicit workload; this default -/// fixture is the one whose digest the package pins. +/// fixture is the one whose input digest the package pins. Ranking receipts +/// are checked separately and are not part of that identity. pub fn validate_default_workload() -> Result { let assets = packaged_assets::materialize()?; validate_direct_workload(assets.root(), Some(&assets.workload_path())) diff --git a/crates/tracedecay-search-eval/src/report_tests.rs b/crates/tracedecay-search-eval/src/report_tests.rs index 9d1aaa508f..fa46c7fdfc 100644 --- a/crates/tracedecay-search-eval/src/report_tests.rs +++ b/crates/tracedecay-search-eval/src/report_tests.rs @@ -38,8 +38,9 @@ fn baseline_report_retains_raw_fallback_current_and_exact_ten_x_samples() { .expect("generate direct fixture outputs"); let report = evaluate_generated_outputs(repo_root, workload, &generated) .expect("evaluate direct fixture outputs"); - // Production retrieval changes must land with a re-pinned workload; the - // pin is what turns a silent ranking change into a visible one. + // A ranking change must move the receipt. A generation reseal must not: + // the receipt hashes ordered rows and lane coverage, not the sealed + // generation those rows were bound under. for profile in &report.profiles { let observed = generated .outputs @@ -56,11 +57,12 @@ fn baseline_report_retains_raw_fallback_current_and_exact_ten_x_samples() { .unwrap_or_else(|| "no generated output for this profile".to_owned()); assert!( profile.fallback_matches_expected, - "{}:{} query fallback digest drifted from \ + "{}:{} ranking receipt drifted from \ `expected_query_fallback_digests.{}` in \ - tests/fixtures/search_quality/query-lexical-graph-workload-v1.json \ - ({observed}). Confirm the new query results are intended, then re-pin \ - the packaged workload, packaged::WORKLOAD_SHA256, and the workload digest pins.", + crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json \ + ({observed}). The receipt binds ordered ranking rows and lane coverage, \ + not generation or extractor-revision identity. Re-pin only that receipt \ + when the ranking itself changed; do not touch the workload identity pin.", profile.profile_id, profile.partition, profile.partition ); } From f51da582b355dadeb0ea994f9804052bd61bf379 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:37:57 +0000 Subject: [PATCH 24/84] fix(store): one owner for a reached cursor frontier Live ingest and catch-up both advance the same coverage with different reasons. The loser was a permanent cursor-advance collision, so both owners warned on every retry. A frontier the durable cursor has already reached keeps the first ledger row and returns an exact duplicate. A disagreement is still a write failure when the cursor has not moved. Co-authored-by: Zack Jackson --- crates/tracedecay-domain/src/observation.rs | 13 + .../src/observation_adapter.rs | 34 ++- .../src/observation_collision_tests.rs | 233 ++++++++++++++++-- .../src/repository/observation/mod.rs | 20 +- .../src/repository/observation/tests.rs | 68 ++++- .../src/daemon/store_runtime_tests.rs | 12 +- .../session_suite/observation_store/mod.rs | 40 ++- 7 files changed, 365 insertions(+), 55 deletions(-) diff --git a/crates/tracedecay-domain/src/observation.rs b/crates/tracedecay-domain/src/observation.rs index f9fca1fc0e..954855587e 100644 --- a/crates/tracedecay-domain/src/observation.rs +++ b/crates/tracedecay-domain/src/observation.rs @@ -878,6 +878,19 @@ impl ObservationSourceCursorV1 { } Ok(self.byte_offset.cmp(&other.byte_offset)) } + + /// Whether this cursor already owns `frontier` on the same ordering authority. + /// + /// Progress is the position. Resume fingerprints are checkpoints, not + /// coverage, so two owners of the same bytes can disagree there without + /// either being behind the frontier. + #[must_use] + pub fn reached(&self, frontier: &Self) -> bool { + matches!( + self.checked_cmp(frontier), + Ok(Ordering::Equal | Ordering::Greater) + ) + } } pub const CANONICAL_OBSERVATION_ENVELOPE_VERSION_V1: u16 = 1; diff --git a/crates/tracedecay-global-db/src/observation_adapter.rs b/crates/tracedecay-global-db/src/observation_adapter.rs index 8148f6b9e1..df5263e2b1 100644 --- a/crates/tracedecay-global-db/src/observation_adapter.rs +++ b/crates/tracedecay-global-db/src/observation_adapter.rs @@ -1485,8 +1485,16 @@ impl ObservationStore for GlobalDbObservationStore { advance.next_cursor().source(), advance.next_cursor().scope(), )?; - let existed_at_next = actual_cursor.as_ref() == Some(advance.next_cursor()); - if !existed_at_next && actual_cursor.as_ref() != advance.expected_cursor() { + // One owner per frontier. A cursor that already reached `next` has + // recorded the range; a second reason must not become a permanent + // collision that both ingest owners then warn on forever. + if actual_cursor + .as_ref() + .is_some_and(|cursor| cursor.reached(advance.next_cursor())) + { + return Ok(CursorAdvanceOutcome::ExactDuplicate); + } + if actual_cursor.as_ref() != advance.expected_cursor() { return Err(ObservationStoreError::CursorConflict { expected: Box::new(advance.expected_cursor().cloned()), actual: Box::new(actual_cursor), @@ -1498,6 +1506,7 @@ impl ObservationStore for GlobalDbObservationStore { "coverage": advance.coverage(), }); let key = format!("cursor.{}", canonical_runtime_digest(&identity)?); + let next_cursor = advance.next_cursor().clone(); let payload = RepositoryWritePayloadV1::ObservationCursorAdvance(Box::new(advance)); let (command_bytes, command_digest) = canonical_json_bytes_and_sha256( &runtime_command_value(&payload)?, @@ -1515,19 +1524,26 @@ impl ObservationStore for GlobalDbObservationStore { ) .await; match outcome? { - RuntimeSubmitOutcomeV1::Committed { .. } - | RuntimeSubmitOutcomeV1::CommittedAfterCancellation { .. } - if existed_at_next => - { - Ok(CursorAdvanceOutcome::ExactDuplicate) - } RuntimeSubmitOutcomeV1::Committed { .. } | RuntimeSubmitOutcomeV1::CommittedAfterCancellation { .. } => { Ok(CursorAdvanceOutcome::Committed) } RuntimeSubmitOutcomeV1::ExactReplay { .. } => Ok(CursorAdvanceOutcome::ExactDuplicate), + // The other owner committed this coverage key between the + // pre-check and the writer lookup. If the frontier moved, that + // owner already holds the range; the different command digest is + // not a durable collision. RuntimeSubmitOutcomeV1::IdempotencyConflict { .. } => { - Err(ObservationStoreError::CursorAdvanceCollision) + let raced = + read_runtime_source_cursor(runtime, next_cursor.source(), next_cursor.scope())?; + if raced + .as_ref() + .is_some_and(|cursor| cursor.reached(&next_cursor)) + { + Ok(CursorAdvanceOutcome::ExactDuplicate) + } else { + Err(ObservationStoreError::CursorAdvanceCollision) + } } other => Err(runtime_storage_error( "advance observation source cursor", diff --git a/crates/tracedecay-global-db/src/observation_collision_tests.rs b/crates/tracedecay-global-db/src/observation_collision_tests.rs index 31cd26d4db..d98c563493 100644 --- a/crates/tracedecay-global-db/src/observation_collision_tests.rs +++ b/crates/tracedecay-global-db/src/observation_collision_tests.rs @@ -55,9 +55,10 @@ use tracedecay_domain::{ use tracedecay_store::observation::ObservationIdentityCollisionDispositionV1; use tracedecay_store::{ AnchoredObservationWrite, CursorAdvanceLedgerReasonV1, CursorAdvanceLedgerReceiptIdV1, - ObservationCoverageReason, ObservationCursorAdvance, ObservationPersistOutcome, - ObservationProjectionStore, ObservationStore, ObservationStoreError, ObservationWrite, - ProjectionPersistOutcome, ProjectionSkipReason, SESSION_MESSAGE_PROJECTOR_VERSION, + CursorAdvanceOutcome, ObservationCoverageReason, ObservationCursorAdvance, + ObservationPersistOutcome, ObservationProjectionStore, ObservationStore, ObservationStoreError, + ObservationWrite, ProjectionPersistOutcome, ProjectionSkipReason, + SESSION_MESSAGE_PROJECTOR_VERSION, }; use tracing::field::{Field, Visit}; use tracing::span::{Attributes, Id, Record}; @@ -2974,6 +2975,72 @@ async fn failed_coverage_advance_leaves_no_visible_refusal_marker() { ); } +#[tokio::test] +async fn covered_frontier_keeps_the_first_reason_when_a_second_owner_advances() { + let tmp = TempDir::new().unwrap(); + let runtime = HostAdmissionTestRuntimeV1::profile(tmp.path()) + .await + .unwrap(); + let store = runtime + .observation_store(HostAdmissionScope::Profile) + .unwrap(); + let session_id = SessionId::new("session.cursor-owned-frontier").unwrap(); + let (observation, _) = collision_candidate( + &session_id, + "record.cursor-owned-frontier", + 1, + "owned frontier fixture", + "receipt.cursor-owned-frontier", + None, + ); + let advance = ObservationCursorAdvance::for_ordering( + observation.source().clone(), + observation.scope().clone(), + observation.identity().generation(), + observation.identity().ordering_domain(), + None, + observation.identity().position(), + ObservationCoverageReason::OutOfScope, + ) + .unwrap(); + seed_cursor_replay( + &runtime, + &advance, + Some(ObservationCoverageReason::BlankFrame), + ) + .await; + + assert_eq!( + store.advance_source_cursor(advance.clone()).await.unwrap(), + CursorAdvanceOutcome::ExactDuplicate + ); + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .unwrap(); + let snapshot = database.read_snapshot().await.unwrap(); + let mut rows = snapshot + .query( + "SELECT reason, COUNT(*) FROM source_cursor_advances GROUP BY reason", + (), + ) + .await + .unwrap(); + let row = rows.next().await.unwrap().expect("owned ledger row"); + assert_eq!(row.get::(0).unwrap(), "blank_frame"); + assert_eq!(row.get::(1).unwrap(), 1); + assert!(rows.next().await.unwrap().is_none()); + drop(rows); + assert_eq!( + store + .get_source_cursor(observation.source(), observation.scope()) + .await + .unwrap() + .as_ref() + .map(ObservationSourceCursorV1::position), + Some(advance.next_cursor().position()) + ); +} + #[tokio::test] async fn runtime_cursor_replay_preserves_structured_ledger_disagreement() { let tmp = TempDir::new().unwrap(); @@ -3008,6 +3075,18 @@ async fn runtime_cursor_replay_preserves_structured_ledger_disagreement() { Some(ObservationCoverageReason::BlankFrame), ) .await; + // The seeded cursor already stands at `next`. Pull it back so this + // advance is the write that would move the frontier, where a stored + // reason still disagrees. + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .unwrap(); + let transaction = database.begin_write_transaction().await.unwrap(); + transaction + .execute("DELETE FROM source_cursors", ()) + .await + .unwrap(); + transaction.commit().await.unwrap(); let error = store .advance_source_cursor(advance.clone()) @@ -3038,7 +3117,7 @@ async fn runtime_cursor_replay_preserves_structured_ledger_disagreement() { } #[tokio::test] -async fn runtime_cursor_replay_without_a_ledger_row_keeps_generic_collision_semantics() { +async fn covered_cursor_without_a_ledger_row_is_already_owned() { let tmp = TempDir::new().unwrap(); let runtime = HostAdmissionTestRuntimeV1::profile(tmp.path()) .await @@ -3067,10 +3146,123 @@ async fn runtime_cursor_replay_without_a_ledger_row_keeps_generic_collision_sema .unwrap(); seed_cursor_replay(&runtime, &advance, None).await; - assert!(matches!( - store.advance_source_cursor(advance).await.unwrap_err(), - ObservationStoreError::CursorAdvanceCollision - )); + assert_eq!( + store.advance_source_cursor(advance).await.unwrap(), + CursorAdvanceOutcome::ExactDuplicate + ); +} + +#[tokio::test] +async fn concurrent_cursor_owners_with_different_reasons_share_one_frontier() { + let tmp = TempDir::new().unwrap(); + let runtime = HostAdmissionTestRuntimeV1::profile(tmp.path()) + .await + .unwrap(); + let store = runtime + .observation_store(HostAdmissionScope::Profile) + .unwrap(); + let session_id = SessionId::new("session.cursor-concurrent-owners").unwrap(); + let (observation, _) = collision_candidate( + &session_id, + "record.cursor-concurrent-owners", + 1, + "concurrent owners fixture", + "receipt.cursor-concurrent-owners", + None, + ); + let blank = ObservationCursorAdvance::for_ordering( + observation.source().clone(), + observation.scope().clone(), + observation.identity().generation(), + observation.identity().ordering_domain(), + None, + observation.identity().position(), + ObservationCoverageReason::BlankFrame, + ) + .unwrap(); + let out_of_scope = ObservationCursorAdvance::for_ordering( + observation.source().clone(), + observation.scope().clone(), + observation.identity().generation(), + observation.identity().ordering_domain(), + None, + observation.identity().position(), + ObservationCoverageReason::OutOfScope, + ) + .unwrap(); + // Hold the writer so both owners pass the pre-check against the empty + // frontier and only then race the same coverage key. A short-circuit + // after one has already committed would not exercise the conflict path. + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .unwrap(); + let gate = database.begin_write_transaction().await.unwrap(); + let left_store = store.clone(); + let right_store = store.clone(); + let mut left_task = tokio::spawn(async move { left_store.advance_source_cursor(blank).await }); + let mut right_task = + tokio::spawn(async move { right_store.advance_source_cursor(out_of_scope).await }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(200), &mut left_task) + .await + .is_err(), + "the blank-frame owner must wait behind the held writer" + ); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(200), &mut right_task) + .await + .is_err(), + "the out-of-scope owner must wait behind the held writer" + ); + gate.rollback().await.unwrap(); + let left = left_task + .await + .unwrap() + .expect("blank-frame owner must not collide"); + let right = right_task + .await + .unwrap() + .expect("out-of-scope owner must not collide"); + let outcomes = [left, right]; + assert_eq!( + outcomes + .iter() + .filter(|outcome| **outcome == CursorAdvanceOutcome::Committed) + .count(), + 1, + "exactly one owner commits the frontier, got {outcomes:?}" + ); + assert_eq!( + outcomes + .iter() + .filter(|outcome| **outcome == CursorAdvanceOutcome::ExactDuplicate) + .count(), + 1, + "the other owner observes the owned frontier, got {outcomes:?}" + ); + assert_eq!(table_count(&runtime, "source_cursor_advances").await, 1); + assert_eq!(table_count(&runtime, "source_cursors").await, 1); + let cursor = only_source_cursor(&runtime).await; + assert_eq!(cursor.position(), observation.identity().position().end()); + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .unwrap(); + let snapshot = database.read_snapshot().await.unwrap(); + let mut rows = snapshot + .query("SELECT reason FROM source_cursor_advances", ()) + .await + .unwrap(); + let reason = rows + .next() + .await + .unwrap() + .expect("one ledger reason") + .get::(0) + .unwrap(); + assert!( + reason == "blank_frame" || reason == "out_of_scope", + "the retained reason must be one of the two owners, got {reason}" + ); } #[tokio::test] @@ -3103,14 +3295,25 @@ async fn runtime_cursor_replay_preserves_storage_failure() { .unwrap(); seed_cursor_replay(&runtime, &advance, None).await; - assert!(matches!( - store - .advance_source_cursor(advance.clone()) - .await - .unwrap_err(), - ObservationStoreError::CursorAdvanceCollision - )); + assert_eq!( + store.advance_source_cursor(advance.clone()).await.unwrap(), + CursorAdvanceOutcome::ExactDuplicate + ); + let advance = ObservationCursorAdvance::for_ordering( + ObservationSourceIdentityV1::for_provider( + ProviderId::new(COLLISION_PROVIDER).unwrap(), + SessionId::new("session.cursor-runtime-storage-uncovered").unwrap(), + ) + .unwrap(), + ObservationScopeV1::Profile, + observation.identity().generation(), + observation.identity().ordering_domain(), + None, + observation.identity().position(), + ObservationCoverageReason::OutOfScope, + ) + .unwrap(); let database = runtime .registered_database(HostAdmissionScope::Profile) .unwrap(); diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/observation/mod.rs b/crates/tracedecay-rusqlite-runtime/src/repository/observation/mod.rs index 1742ad8d6b..0ff4734612 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/observation/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/observation/mod.rs @@ -177,16 +177,16 @@ impl ObservationExecutor { let source_json = encode(advance.next_cursor().source())?; let scope_json = encode(advance.next_cursor().scope())?; let actual_cursor = read_cursor(savepoint, &source_json, &scope_json)?; - if actual_cursor.as_ref() == Some(advance.next_cursor()) { - if let Some(disagreement) = - cursor_advance_ledger_disagreement(savepoint, &source_json, &scope_json, advance)? - { - return Err(disagreement); - } - if cursor_advance_receipt_matches(savepoint, &source_json, &scope_json, advance)? { - return Ok(()); - } - return Err(StorageOperationError::ObservationCursorAdvanceCollision); + // The durable cursor owns the range. Live ingest and catch-up both + // advance the same bytes with legitimately different reasons; once + // the frontier is reached the first ledger row stays and the later + // owner is a no-op. A disagreement is still a failure below, when + // this advance would be the write that moves the cursor. + if actual_cursor + .as_ref() + .is_some_and(|cursor| cursor.reached(advance.next_cursor())) + { + return Ok(()); } if actual_cursor.as_ref() != advance.expected_cursor() { return Err(observation_source_cursor_conflict( diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/observation/tests.rs b/crates/tracedecay-rusqlite-runtime/src/repository/observation/tests.rs index dd19c6c519..6079a44a5f 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/observation/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/observation/tests.rs @@ -478,6 +478,20 @@ fn execute_cursor_advance( Ok(()) } +fn source_cursor_json(connection: &Connection) -> String { + connection + .query_row("SELECT cursor_json FROM source_cursors", [], |row| { + row.get(0) + }) + .unwrap() +} + +fn restore_source_cursor(connection: &Connection, cursor_json: &str) { + connection + .execute("UPDATE source_cursors SET cursor_json = ?1", [cursor_json]) + .unwrap(); +} + #[test] fn anchored_write_persists_all_authority_rows_atomically() { let mut connection = connection(); @@ -738,10 +752,11 @@ fn identity_collision_fails_without_advancing_the_source_cursor() { } #[test] -fn source_cursor_advance_replays_exactly_and_reports_ledger_disagreement() { +fn source_cursor_advance_keeps_the_first_owner_once_the_frontier_is_reached() { let mut connection = connection(); let write = anchored_observation_write("fixture", "receipt.fixture"); execute(&mut connection, &write).unwrap(); + let owned_frontier = source_cursor_json(&connection); let advance = ObservationCursorAdvance::for_ordering( write.observation().source().clone(), write.observation().scope().clone(), @@ -774,9 +789,30 @@ fn source_cursor_advance_replays_exactly_and_reports_ledger_disagreement() { ObservationCoverageReason::OutOfScope, ) .unwrap(); + execute_cursor_advance(&mut connection, &conflicting).unwrap(); + assert_eq!( + connection + .query_row("SELECT reason FROM source_cursor_advances", [], |row| { + row.get::<_, String>(0) + },) + .unwrap(), + ObservationCoverageReason::BlankFrame.as_str() + ); + assert_eq!( + connection + .query_row("SELECT COUNT(*) FROM source_cursor_advances", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 1 + ); + + // The disagreement stays a write-time failure: the cursor has not + // reached the proposed frontier, so this advance would move it. + restore_source_cursor(&connection, &owned_frontier); let error = execute_cursor_advance(&mut connection, &conflicting).unwrap_err(); let StorageOperationError::CursorAdvanceLedgerDisagreement { disagreement } = error else { - panic!("expected structured immutable ledger disagreement"); + panic!("expected structured immutable ledger disagreement, got {error:?}"); }; assert_eq!(disagreement.source(), write.observation().source()); assert_eq!(disagreement.scope(), write.observation().scope()); @@ -804,6 +840,7 @@ fn canonical_cursor_advance_receipt_remains_typed_after_authority_lookup() { let mut connection = connection(); let write = anchored_observation_write("fixture", "receipt.fixture"); execute(&mut connection, &write).unwrap(); + let observation_cursor = source_cursor_json(&connection); let advance = ObservationCursorAdvance::for_ordering_with_sanitization_receipt( write.observation().source().clone(), write.observation().scope().clone(), @@ -829,9 +866,26 @@ fn canonical_cursor_advance_receipt_remains_typed_after_authority_lookup() { ) .unwrap(); + execute_cursor_advance(&mut connection, &conflicting).unwrap(); + assert_eq!( + connection + .query_row( + "SELECT reason, receipt_id FROM source_cursor_advances", + [], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .unwrap(), + ( + ObservationCoverageReason::DuplicateObservation + .as_str() + .to_owned(), + "receipt.fixture".to_owned(), + ) + ); + restore_source_cursor(&connection, &observation_cursor); let error = execute_cursor_advance(&mut connection, &conflicting).unwrap_err(); let StorageOperationError::CursorAdvanceLedgerDisagreement { disagreement } = error else { - panic!("expected structured immutable ledger disagreement"); + panic!("expected structured immutable ledger disagreement, got {error:?}"); }; assert!(matches!( disagreement.stored().receipt_id(), @@ -850,6 +904,7 @@ fn corrupt_cursor_advance_ledger_values_are_opaque_and_content_free() { let mut connection = connection(); let write = anchored_observation_write("fixture", "receipt.fixture"); execute(&mut connection, &write).unwrap(); + let observation_cursor = source_cursor_json(&connection); let advance = ObservationCursorAdvance::for_ordering( write.observation().source().clone(), write.observation().scope().clone(), @@ -879,10 +934,11 @@ fn corrupt_cursor_advance_ledger_values_are_opaque_and_content_free() { ObservationCoverageReason::OutOfScope, ) .unwrap(); + restore_source_cursor(&connection, &observation_cursor); let error = execute_cursor_advance(&mut connection, &conflicting).unwrap_err(); let StorageOperationError::CursorAdvanceLedgerDisagreement { disagreement } = error else { - panic!("expected structured immutable ledger disagreement"); + panic!("expected structured immutable ledger disagreement, got {error:?}"); }; assert!(matches!( disagreement.stored().reason(), @@ -912,6 +968,7 @@ fn short_corrupt_ledger_receipt_stays_opaque_across_runtime_boundary() { let mut connection = connection(); let write = anchored_observation_write("fixture", "receipt.fixture"); execute(&mut connection, &write).unwrap(); + let observation_cursor = source_cursor_json(&connection); let advance = ObservationCursorAdvance::for_ordering( write.observation().source().clone(), write.observation().scope().clone(), @@ -942,10 +999,11 @@ fn short_corrupt_ledger_receipt_stays_opaque_across_runtime_boundary() { ObservationCoverageReason::OutOfScope, ) .unwrap(); + restore_source_cursor(&connection, &observation_cursor); let error = execute_cursor_advance(&mut connection, &conflicting).unwrap_err(); let StorageOperationError::CursorAdvanceLedgerDisagreement { disagreement } = error else { - panic!("expected structured immutable ledger disagreement"); + panic!("expected structured immutable ledger disagreement, got {error:?}"); }; assert!(matches!( disagreement.stored().receipt_id(), diff --git a/crates/tracedecay/src/daemon/store_runtime_tests.rs b/crates/tracedecay/src/daemon/store_runtime_tests.rs index 3b58371f8a..b60d6f0f74 100644 --- a/crates/tracedecay/src/daemon/store_runtime_tests.rs +++ b/crates/tracedecay/src/daemon/store_runtime_tests.rs @@ -30,7 +30,7 @@ use tracedecay_session_memory::memory::{ }; use tracedecay_store::{ CursorAdvanceOutcome, FactReadControl, FactWriteControl, ObservationCoverageReason, - ObservationCursorAdvance, ObservationStore, ObservationStoreError, ProjectId, + ObservationCursorAdvance, ObservationStore, ProjectId, ProjectMemoryFactHistoryQueryV1, ProjectMemoryFactIdV1, ProjectMemoryFactProjectionV1, RetainedGraphStoreLeaseV1, StoreShardIdV1, }; @@ -1198,14 +1198,14 @@ async fn retained_runtime_ledger_replays_during_bounded_background_convergence() "retired", ObservationCoverageReason::BlankFrame, ); - assert!(matches!( + assert_eq!( database .observation_store() .advance_source_cursor(conflicting_advance) .await - .expect_err("classify retained cursor collision while convergence is pending"), - ObservationStoreError::CursorAdvanceCollision - )); + .expect("a later reason does not unseat the owned frontier"), + CursorAdvanceOutcome::ExactDuplicate + ); let fresh_advance = runtime_cursor_advance(&project_id, "fresh", ObservationCoverageReason::OutOfScope); @@ -1258,7 +1258,7 @@ async fn retained_runtime_ledger_replays_during_bounded_background_convergence() .get::(0) .expect("decode committed cursor effect count"), 2, - "the retained replay and collision must not create another cursor effect" + "the retained replay and the later reason must not create another cursor effect" ); let mut receipts = snapshot .query( diff --git a/crates/tracedecay/tests/session_suite/observation_store/mod.rs b/crates/tracedecay/tests/session_suite/observation_store/mod.rs index c4d27abd67..a8e3e542af 100644 --- a/crates/tracedecay/tests/session_suite/observation_store/mod.rs +++ b/crates/tracedecay/tests/session_suite/observation_store/mod.rs @@ -1476,12 +1476,16 @@ async fn cursor_only_progress_persists_non_payload_receipt_and_retries_idempoten } #[tokio::test] -async fn cursor_only_retry_rejects_same_cursor_with_different_reason() { +async fn cursor_already_owned_keeps_the_first_reason_for_a_later_owner() { let tmp = TempDir::new().unwrap(); let runtime = profile_runtime(&tmp).await; let store = runtime .observation_store(HostAdmissionScope::Profile) .unwrap(); + let database_path = runtime + .database_path(HostAdmissionScope::Profile) + .unwrap() + .to_path_buf(); store .advance_source_cursor(cursor_advance( @@ -1493,7 +1497,7 @@ async fn cursor_only_retry_rejects_same_cursor_with_different_reason() { .await .unwrap(); - assert!(matches!( + assert_eq!( store .advance_source_cursor(cursor_advance( None, @@ -1501,17 +1505,32 @@ async fn cursor_only_retry_rejects_same_cursor_with_different_reason() { 10, NonDurableFrameReason::OutOfScope, )) - .await, - Err(ObservationStoreError::CursorAdvanceCollision) - )); + .await + .unwrap(), + CursorAdvanceOutcome::ExactDuplicate + ); assert_eq!( store.get_source_cursor(&source(), &scope()).await.unwrap(), Some(cursor(10)) ); + let conn = rusqlite::Connection::open(&database_path).unwrap(); + let reason: String = conn + .query_row("SELECT reason FROM source_cursor_advances", (), |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(reason, "blank_frame"); + assert_eq!( + conn.query_row("SELECT COUNT(*) FROM source_cursor_advances", (), |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 1 + ); } #[tokio::test] -async fn cursor_only_retry_rejects_same_cursor_with_different_coverage() { +async fn cursor_already_past_a_narrower_range_keeps_the_owned_frontier() { let tmp = TempDir::new().unwrap(); let runtime = profile_runtime(&tmp).await; let store = runtime @@ -1528,7 +1547,7 @@ async fn cursor_only_retry_rejects_same_cursor_with_different_coverage() { .await .unwrap(); - assert!(matches!( + assert_eq!( store .advance_source_cursor(cursor_advance( Some(cursor(5)), @@ -1536,9 +1555,10 @@ async fn cursor_only_retry_rejects_same_cursor_with_different_coverage() { 10, NonDurableFrameReason::BlankFrame, )) - .await, - Err(ObservationStoreError::CursorAdvanceCollision) - )); + .await + .unwrap(), + CursorAdvanceOutcome::ExactDuplicate + ); assert_eq!( store.get_source_cursor(&source(), &scope()).await.unwrap(), Some(cursor(10)) From e74cb00373f8c9a133d2e502037ab7c023f3eaf5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:43:41 +0000 Subject: [PATCH 25/84] fix(code-index): stop busy reads pinning freshness Verifying A query that cannot join the scheduler posted BusyFollowUp whenever the proof had expired. The in-flight pass consumes that arrival, the next poll finds the lock still held, and posts another. Dashboard freshness treats the pending slot as refresh_in_flight and stays Verifying. The holder is already the observation; the next read that acquires the scheduler still requests a proof that remains expired. Co-authored-by: Zack Jackson --- .../registry/serving_reads.rs | 22 ++- .../code_index_scheduler/tests/reconcile.rs | 138 ++++++++++++++++++ .../ignored_dependency_admission_tests.rs | 13 +- 3 files changed, 153 insertions(+), 20 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/serving_reads.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/serving_reads.rs index 668576b8d5..2c9a5b0a36 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/serving_reads.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/serving_reads.rs @@ -564,18 +564,16 @@ impl CodeIndexSchedulerRegistryV1 { .read() .unwrap_or_else(std::sync::PoisonError::into_inner) .clone(); - // A still-current proof needs no follow-up. If it expired - // after this pass began, leave one coalesced wake so the - // worker re-observes source after releasing its ownership. - if serving.is_some() - && !source_freshness.ready_without_stat(&freshness_root, &shutting_down) - { - Self::note_wake_if_idle( - &pending_wake, - &wake, - CodeIndexCadenceTriggerV1::BusyFollowUp, - ); - } + // The holder of the scheduler is already the source + // observation. A follow-up posted from this read is taken + // by that pass, the slot goes empty, and the next poll + // finds the lock still held with the proof not yet + // renewed and posts another. Dashboard freshness reads + // that slot as `refresh_in_flight` and stays `Verifying` + // for the whole chain. The pass renews the proof before + // it releases the lock; a proof that is still expired + // afterwards is requested by the next read that acquires + // the scheduler. return serving; } }; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs index 138157469d..1bf96581b1 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs @@ -3772,6 +3772,144 @@ async fn dashboard_progress_does_not_wait_for_the_scheduler_mutex() { registry.shutdown().await; } +/// A query that cannot join the owner must not schedule the verification the +/// dashboard would then report as `Verifying`. The in-flight pass renews an +/// expired proof before it releases the scheduler; a read that posts +/// `BusyFollowUp` while that pass holds the lock is taken and immediately +/// replaced by the next poll, so the ladder never settles to `Fresh`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn busy_query_does_not_rearm_dashboard_verification() { + let fixture = GitFixture::new(&[("src/main.rs", "fn main() {}\n")]); + let store = TempDir::new().expect("store root"); + let registry = CodeIndexSchedulerRegistryV1::with_background_reconcile_permits(1, 1); + registry + .mount_worktree( + test_project_id(), + fixture.path(), + store.path().to_path_buf(), + ) + .await + .expect("mount daemon-owned scheduler"); + wait_for_initial_generation(®istry, fixture.path()).await; + wait_for_dashboard_ready(®istry, fixture.path()).await; + drain_clone_backfill(®istry, fixture.path()).await; + settled_owner_with_idle_admission(®istry, fixture.path()).await; + let admission = quiesced_background_reconcile_admission(®istry, fixture.path()).await; + let canonical_root = fixture + .path() + .canonicalize() + .expect("canonical fixture root"); + let scope = { + let mounted = registry.mounted.lock().await; + let worktree = mounted.get(&canonical_root).expect("mounted worktree"); + tracedecay_contracts::ResolvedScope::new( + test_project_id(), + worktree.repository_id.clone(), + worktree.worktree_id.clone(), + None, + ) + .expect("resolved scope") + }; + clear_pending_wake_until_quiet(®istry, &scope).await; + let freshness = registry + .source_freshness_for_root(fixture.path()) + .await + .expect("mounted freshness fence"); + { + let mut state = freshness.state.lock().expect("freshness state"); + state.last_reconciled_at = Instant::now() + .checked_sub(state.staleness_threshold + Duration::from_secs(1)) + .expect("age the readiness proof"); + } + let scheduler = { + let mounted = registry.mounted.lock().await; + Arc::clone( + &mounted + .get(&canonical_root) + .expect("mounted worktree") + .scheduler, + ) + }; + let (locked_tx, locked_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let scheduler_holder = tokio::task::spawn_blocking(move || { + let _scheduler_guard = scheduler.lock().expect("hold scheduler mutex"); + let _ = locked_tx.send(()); + let _ = release_rx.blocking_recv(); + }); + locked_rx.await.expect("scheduler mutex holder started"); + for _ in 0..8 { + assert!( + registry + .latest_complete_fresh(fixture.path()) + .await + .is_some(), + "a busy owner still serves the seated generation" + ); + } + assert_eq!( + registry.pending_wake_micros_for_root(fixture.path()).await, + Some(0), + "a read blocked on the in-flight owner must not schedule another verification" + ); + let projected = registry + .dashboard_freshness(fixture.path()) + .await + .expect("dashboard freshness while the owner holds the scheduler"); + assert_eq!( + projected.staleness_state, + Some(tracedecay_contracts::code_index_freshness::CodeIndexStalenessStateV1::Fresh), + "an owner that has not observed a source change is not Verifying" + ); + let _ = release_tx.send(()); + scheduler_holder + .await + .expect("scheduler mutex holder joined"); + + assert!( + registry + .latest_complete_fresh(fixture.path()) + .await + .is_some(), + "the seated generation remains servable once the owner releases the scheduler" + ); + assert!( + registry + .pending_wake_micros_for_root(fixture.path()) + .await + .is_some_and(|pending| pending != 0), + "an uncontended read of an expired proof still requests one verification" + ); + drop(admission); + tokio::time::timeout(SERVING_SEAT_FAILURE_CEILING, async { + loop { + let settled = registry + .dashboard_freshness(fixture.path()) + .await + .is_some_and(|freshness| { + freshness.staleness_state + == Some( + tracedecay_contracts::code_index_freshness::CodeIndexStalenessStateV1::Fresh, + ) + }) + && registry + .pending_wake_micros_for_root(fixture.path()) + .await + == Some(0) + && !registry + .reconcile_in_progress_for_test(fixture.path()) + .await; + if settled { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("the single verification settles back to Fresh"); + registry.shutdown().await; +} + // Two workers so the timeout timer stays live if a regression parks one // runtime worker on the scheduler mutex: the test then fails instead of // deadlocking against its own release channel. diff --git a/crates/tracedecay/src/daemon/project_open_owners/code_index_reads/ignored_dependency_admission_tests.rs b/crates/tracedecay/src/daemon/project_open_owners/code_index_reads/ignored_dependency_admission_tests.rs index 946f6bf383..2b72305215 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/code_index_reads/ignored_dependency_admission_tests.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/code_index_reads/ignored_dependency_admission_tests.rs @@ -388,16 +388,13 @@ async fn latest( // Lightweight publication precedes complete-generation seating. Demand // that complete state before using its imports as admission evidence. The // seat is background work behind the scheduler mutex; under a loaded CI - // runner it has taken over 5 s, so the bound is a minute. - // Poll the dashboard projection alone while the owner is busy: the - // query-admission read (`latest_complete_fresh`) leaves a coalesced wake - // behind whenever it finds the worker holding the scheduler with an - // expired proof, and polling it every 25 ms re-armed a no-op pass faster - // than the ladder could settle to `Fresh` (CI run 35419627712: one - // minute of `Verifying`, then 0.3 s on the retry). Read the generation - // only once the ladder has settled. + // runner it has taken over 5 s, so the bound is a minute. Polling the + // query read is safe: a read that finds the owner holding the scheduler + // does not schedule the successor the dashboard would project as + // `Verifying`. tokio::time::timeout(Duration::from_mins(1), async { loop { + let _ = registry.latest_complete_fresh(project_root).await; if registry .dashboard_freshness(project_root) .await From 6bdb930c35ccb0d97ee419c41ec2073ab9291b0e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:44:15 +0000 Subject: [PATCH 26/84] test(mcp): keep the seeded message for describe asserts The handler test moved the seed string into ingest, so the length assertion could not read it. Co-authored-by: Zack Jackson --- crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_test.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_test.rs index 6f6112c323..5e02ef225c 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_test.rs @@ -65,7 +65,8 @@ async fn lcm_session_handlers_expose_bounded_read_apis_and_placeholders() { let (cg, _env) = init_test_project(dir.path()).await; let full_text = format!("orchard dispatch {}", "external-payload-body ".repeat(220)); let projection = - seed_temporal_lcm_session_message(&cg, "lcm-session", "lcm-message", full_text, 1).await; + seed_temporal_lcm_session_message(&cg, "lcm-session", "lcm-message", full_text.clone(), 1) + .await; let temporal_db = open_active_project_session_db(&cg).await; activate_test_temporal_generation(&temporal_db, "lcm-session", vec![projection]).await; let db = open_active_project_session_db(&cg).await; From 3034d5ad79930279533f4a4505a177960febdf9f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:44:24 +0000 Subject: [PATCH 27/84] test(daemon): retire the listen socket with its owner SIGKILL skips the daemon's endpoint unlink. A child that inherited the listen descriptor still accepts on that path, so the next spawn reports a live daemon. branch_search init does exactly that: spawn, init, drop, spawn again. Waiting for the path to go quiet papers over the leak. Kill the owner's process group, then unlink the socket the owner bound. A path that still accepts is a daemon this call did not stop. Co-authored-by: Zack Jackson --- crates/tracedecay/tests/common/mod.rs | 127 +++++++++++------- .../daemon_fault_harness_test.rs | 69 ++++++++++ 2 files changed, 148 insertions(+), 48 deletions(-) diff --git a/crates/tracedecay/tests/common/mod.rs b/crates/tracedecay/tests/common/mod.rs index 28bbc596c4..624a492e20 100644 --- a/crates/tracedecay/tests/common/mod.rs +++ b/crates/tracedecay/tests/common/mod.rs @@ -657,6 +657,9 @@ pub fn http_agent_with_timeout(timeout: Duration) -> ureq::Agent { /// panic while the child is still running, `Drop` force-stops and reaps it. pub struct TestChildProcess { child: Child, + /// Unix socket this child bound. Retired once the child is reaped so a + /// descriptor a subprocess inherited cannot keep the path connectable. + owned_unix_endpoint: Option, } /// Daemon-specific name retained for test fixtures that keep a daemon alive. @@ -664,7 +667,28 @@ pub type DaemonProcess = TestChildProcess; impl TestChildProcess { pub fn new(child: Child) -> Self { - Self { child } + Self { + child, + owned_unix_endpoint: None, + } + } + + /// Record the Unix socket this child bound. + /// + /// `SIGKILL` skips the daemon's own endpoint cleanup (`drop(listener)` then + /// unlink). A subprocess that inherited the listen descriptor across `fork` + /// still accepts on that path after the owner is reaped, which is a leaked + /// descriptor, not a live daemon. Retiring the path with the owner is what + /// the graceful shutdown already does, and what a forced stop must do too. + #[cfg(unix)] + pub fn own_unix_endpoint(&mut self, path: PathBuf) { + self.owned_unix_endpoint = Some(path); + } + + fn retire_owned_unix_endpoint(&mut self) { + if let Some(path) = self.owned_unix_endpoint.take() { + let _ = std::fs::remove_file(path); + } } pub fn id(&self) -> u32 { @@ -752,9 +776,12 @@ impl TestChildProcess { /// Force-stops the daemon and reaps its process before returning. /// /// `Child::kill` maps to `SIGKILL` on Unix and the platform termination - /// primitive elsewhere, keeping fault-injection tests portable. + /// primitive elsewhere, keeping fault-injection tests portable. The owned + /// Unix endpoint, when recorded, is unlinked only after that reap. pub fn kill_and_wait(&mut self) -> std::io::Result { - terminate_and_reap(&mut self.child) + let status = terminate_and_reap(&mut self.child)?; + self.retire_owned_unix_endpoint(); + Ok(status) } fn drain_stderr(&mut self) { @@ -778,15 +805,29 @@ impl TestChildProcess { impl Drop for TestChildProcess { fn drop(&mut self) { - let _ = terminate_and_reap(&mut self.child); + if terminate_and_reap(&mut self.child).is_ok() { + self.retire_owned_unix_endpoint(); + } } } -/// PID-directed stop: survives `process_group(0)` / `setsid` detachment. +/// Stop the child and the process group it leads. +/// +/// Test daemons are started with `process_group(0)`, so the spawned pid is the +/// group leader and is outside the test's group. `Child::kill` signals only +/// that pid. A child still between `fork` and `exec` inherits the listen +/// descriptor and stays in the leader's group, so the path keeps accepting +/// after the leader is reaped. Signal that group first, while the pid is still +/// the live leader, then reap the leader. Do not signal after `try_wait` has +/// reaped: the pid can be recycled. A child that left the group (its own +/// `process_group(0)`) is not this signal's target; unlinking the owned socket +/// retires the path those descriptors were bound to. fn terminate_and_reap(child: &mut Child) -> std::io::Result { + let pid = child.id(); if let Ok(Some(status)) = child.try_wait() { return Ok(status); } + signal_owned_process_group(pid); if let Err(kill_err) = child.kill() { if let Some(status) = child.try_wait()? { @@ -798,6 +839,23 @@ fn terminate_and_reap(child: &mut Child) -> std::io::Result { child.wait() } +#[cfg(unix)] +fn signal_owned_process_group(pid: u32) { + let Ok(pid) = i32::try_from(pid) else { + return; + }; + if pid <= 0 { + return; + } + // Safety: `pid` is the still-live child returned by `Child::id` before + // `try_wait` reaps it, and `process_group(0)` made that pid the group id. + // A non-positive argument would signal this process's own group. + let _ = unsafe { libc::kill(-pid, libc::SIGKILL) }; +} + +#[cfg(not(unix))] +fn signal_owned_process_group(_pid: u32) {} + /// Detach a test child from the test process group. /// /// Nextest (and other harness timeouts) signal the test's process group. @@ -1089,13 +1147,6 @@ pub fn spawn_tracedecay_daemon_with( spawn_tracedecay_daemon_process(&home, &binary, configure) } -/// How long a replacement daemon waits for a stopped predecessor's endpoint to -/// stop accepting before reporting it as still live. -/// -/// Generous on purpose: the wait only costs time when a predecessor is -/// genuinely still reachable, and a real leak still fails rather than hangs. -const PREDECESSOR_DAEMON_VACATE_TIMEOUT: Duration = Duration::from_secs(10); - fn spawn_tracedecay_daemon_process( home: &Path, binary: &Path, @@ -1118,42 +1169,20 @@ fn spawn_tracedecay_daemon_process( }) .is_some_and(|address| TcpStream::connect(address).is_ok()) }; - // Stopping a predecessor daemon is asynchronous with respect to its - // endpoint: `kill` plus `wait` reaps the PID the harness spawned, but the - // kernel keeps the listening socket alive while *any* duplicate of that - // descriptor survives, including one a subprocess inherited across `fork` - // and still holds because it has not reached its own `exec` yet. Asserting - // instantaneously therefore reports an ordinary teardown tail as a live - // daemon, which is what `init_project_fixture` journeys (spawn, init, drop, - // spawn again) hit on a loaded runner. Wait a bounded time for the endpoint - // to stop accepting; a daemon that keeps accepting still fails with the - // same refusal. - poll_until( - Instant::now() + PREDECESSOR_DAEMON_VACATE_TIMEOUT, - Duration::from_millis(25), - || { - #[cfg(unix)] - let live = std::os::unix::net::UnixStream::connect(&socket_path).is_ok(); - #[cfg(not(unix))] - let live = portable_daemon_connectable(); - (!live).then_some(()) - }, - || { - #[cfg(unix)] - { - format!( - "refusing to replace a live test daemon at {}", - socket_path.display() - ) - } - #[cfg(not(unix))] - { - format!( - "refusing to replace a live test daemon recorded at {}", - authority_path.display() - ) - } - }, + // A reaped predecessor has already unlinked its socket. A path that still + // accepts is a daemon this call did not stop, and replacing it would bind + // over a live owner. + #[cfg(unix)] + assert!( + std::os::unix::net::UnixStream::connect(&socket_path).is_err(), + "refusing to replace a live test daemon at {}", + socket_path.display() + ); + #[cfg(not(unix))] + assert!( + !portable_daemon_connectable(), + "refusing to replace a live test daemon recorded at {}", + authority_path.display() ); let mut command = Command::new(binary); @@ -1169,6 +1198,8 @@ fn spawn_tracedecay_daemon_process( detach_from_test_process_group(&mut command); let child = command.spawn().expect("tracedecay daemon should start"); let mut daemon = DaemonProcess::new(child); + #[cfg(unix)] + daemon.own_unix_endpoint(socket_path.clone()); let deadline = Instant::now() + Duration::from_secs(10); poll_until( diff --git a/crates/tracedecay/tests/transport_acceptance_suite/daemon_fault_harness_test.rs b/crates/tracedecay/tests/transport_acceptance_suite/daemon_fault_harness_test.rs index dcf97a9b1d..447b47d06f 100644 --- a/crates/tracedecay/tests/transport_acceptance_suite/daemon_fault_harness_test.rs +++ b/crates/tracedecay/tests/transport_acceptance_suite/daemon_fault_harness_test.rs @@ -321,6 +321,75 @@ fn configured_daemon_can_be_killed_and_reaped() { } } +/// A listen descriptor inherited across `fork` must not outlive its owner. +/// +/// `branch_search_serves_a_committed_generation_behind_dirty_worktree_state` +/// drops the init daemon and immediately spawns the next one. Killing only the +/// leader leaves the inherited listener accepting on the same path, so the +/// next spawn reports a live daemon. The owner is the group leader; retiring +/// its socket with it makes that path refuse the moment the owner is reaped. +#[cfg(unix)] +#[test] +fn reaped_owner_releases_an_inherited_listen_socket() { + use std::os::unix::process::CommandExt; + use std::process::Stdio; + use std::time::{Duration, Instant}; + + let home = tempdir_or_panic(); + let socket_path = home.path().join("inherited-listen.sock"); + let script = r#" +import os, socket, sys, time +sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) +sock.bind(sys.argv[1]) +sock.listen(1) +if os.fork() == 0: + time.sleep(60) +else: + time.sleep(60) +"#; + let mut command = std::process::Command::new("python3"); + command + .arg("-c") + .arg(script) + .arg(&socket_path) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .process_group(0); + let child = command.spawn().expect("python listener should start"); + let mut owner = common::TestChildProcess::new(child); + owner.own_unix_endpoint(socket_path.clone()); + + let deadline = Instant::now() + Duration::from_secs(5); + loop { + if std::os::unix::net::UnixStream::connect(&socket_path).is_ok() { + break; + } + if let Some(status) = owner + .try_wait() + .expect("listener status should be readable") + { + panic!("listener exited before accepting: {status}"); + } + assert!( + Instant::now() < deadline, + "inherited listener never accepted on {}", + socket_path.display() + ); + std::thread::sleep(Duration::from_millis(20)); + } + + drop(owner); + assert!( + std::os::unix::net::UnixStream::connect(&socket_path).is_err(), + "an inherited listen descriptor must not keep the owner's path accepting" + ); + assert!( + !socket_path.exists(), + "reaping the owner must unlink the socket it bound" + ); +} + #[cfg(all(unix, tracedecay_observation_fault_harness, feature = "test-transport"))] async fn assert_daemon_crash_stage( barrier_stage: &str, From a2f50b7970fed35250eea391d2ef678cb6466aa4 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:46:30 +0000 Subject: [PATCH 28/84] docs(code-index): leave clone copy out of witness rebind The seat-swap witness does not own the lexical full-copy. That copy stays on the retained successor driver. Co-authored-by: Zack Jackson --- .../src/code_index_scheduler/reconcile.rs | 9 ++++----- .../src/code_index_scheduler/registry/mount.rs | 7 ++++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 878c9b53e9..2ba6cbacf4 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -2889,11 +2889,10 @@ impl CodeIndexWorktreeSchedulerV1 { /// Bind a sealed snapshot to the source proof, renewing an expired clock /// when the sealed digests still match. /// - /// The admission window is 30s. A graph seal and the clone-fingerprint - /// backfill both outlive it under load. Treating that expiry as "this - /// generation is not the proof" cleared the serving witness and the next - /// pass resealed the same snapshot. A hook epoch or a digest mismatch - /// still refuses; only an unchanged sealed snapshot keeps its generation. + /// The admission window is 30s. This does not move the clone-successor + /// copy off the publication advance. It only stops an expired clock, or a + /// predecessor disk witness, from clearing the generation those digests + /// already name. A hook epoch or a digest mismatch still refuses. pub(super) fn currency_witness_for_sealed_snapshot( &self, generation_id: &CodeGenerationId, diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs index 0a726b2573..3d91e3a619 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs @@ -1835,9 +1835,10 @@ impl CodeIndexSchedulerRegistryV1 { // has verified *this* sealed snapshot is what makes // the binding truthful for a seat this pass did not // publish. An expired clock, or a git-index sample - // this seal moved, is not a different snapshot: - // dropping the witness here is how a newer - // generation stayed unserved through clone backfill. + // this seal moved, is not a different snapshot. + // Dropping the witness here cleared the newer + // generation. The lexical full-copy is not decided + // on this swap. let sealed_currency = scheduler.currency_witness_for_sealed_snapshot( &latest.generation().manifest().generation_id, &latest.generation().snapshot().content_identity, From 6f7468c626e9851ca78abc728ba02bd5468c39fb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:50:42 +0000 Subject: [PATCH 29/84] fix(global-db): refuse tampered projection messages on reopen A projected message that matches neither this binary nor a shipped release was rewritten as an interrupted write. Repair only the shipped rendering; any other body fails profile reopen and stays untouched. Co-authored-by: Zack Jackson --- .../src/observation_projection/apply.rs | 15 +-- .../src/schema_contract/invariants/audit.rs | 21 +++-- .../invariants/released_rendering.rs | 59 +++++++++++- .../src/canonical_projection.rs | 91 +++++++++++++++++-- crates/tracedecay-store/src/lib.rs | 3 +- .../observation_projection/failure_audit.rs | 40 ++++---- 6 files changed, 190 insertions(+), 39 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation_projection/apply.rs b/crates/tracedecay-global-db/src/observation_projection/apply.rs index a9ad8c7ce1..9c30cd32ab 100644 --- a/crates/tracedecay-global-db/src/observation_projection/apply.rs +++ b/crates/tracedecay-global-db/src/observation_projection/apply.rs @@ -820,13 +820,14 @@ pub(in super::super) enum ConvergedRendering { /// deterministic rendering, keeping the historical `message_created` flag the /// releases wrote. /// -/// Reached only from the authority audit, which has already proven the stored -/// provenance row is the digest of the output row this store holds, the -/// rendering a release wrote, rather than a row disagreeing with its own -/// output. The message row and its LCM raw twin are pure derivations of the -/// durable observation, so rewriting them loses nothing; the digest is -/// re-stamped last so an interrupted transaction leaves the released pairing -/// intact. +/// Reached only from the authority audit, which has already admitted the row +/// as a shipped rendering: provenance still carries the digest of the output +/// this store holds, or it carries this binary's digest while the mutable row +/// is still that shipped rendering. A row that matches neither is refused +/// before this write. The message row and its LCM raw twin are pure +/// derivations of the durable observation, so rewriting them loses nothing; +/// the digest is re-stamped last so an interrupted transaction leaves the +/// released pairing intact. /// /// When the LCM privacy sanitizer withholds this binary's rendering, that /// verdict *is* the current rendering: the output is retired to the disposition diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs index 565c0a309d..64e3b950a9 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs @@ -5,6 +5,7 @@ use tracedecay_domain::DurableObservationV1; use tracedecay_store::{ ObservationProjection, ProjectionSkipReason, ProjectionStoreError, SESSION_MESSAGE_PROJECTOR_VERSION, SessionMessageProjection, WorkflowFactProjection, + stored_message_is_shipped_release_rendering, }; use crate::observation_projection::{ProjectionOutputAuthority, ProjectionRowsBatch}; @@ -834,14 +835,20 @@ async fn validate_message_projection_row( message_id, }) if output_row_present && provider == owner_projection.message().provider - && message_id == owner_projection.message().message_id => + && message_id == owner_projection.message().message_id + && resolved + .projection_rows + .message(&provider, &message_id) + .is_some_and(|stored| { + stored_message_is_shipped_release_rendering(&authority.canonical, stored) + }) => { - // Ownership was validated above, the immutable observation - // re-derived this projection, and its provenance already - // carries the projection's current digest. The mutable output - // row is the only stale member, an interrupted/older write - // shape observed in ProfileSessions. Finish that write in the - // same convergence ledger used for released renderings. + // Provenance already carries this binary's digest, and the + // mutable row is still the rendering a shipped release wrote + // for this observation: the write that stamped the digest did + // not finish. Finish it on the released-rendering ledger. A + // body that matches neither rendering is tamper and falls + // through to the hard failure below. resolved.released.record(&owner_projection); } Err(ProjectionStoreError::SessionOutputCollision { diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs index 760c1dcbd9..2765ed324c 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs @@ -33,7 +33,10 @@ //! that owns the transaction. A row disagreeing on identity, anchor, receipt, //! output provider or message id, or carrying a digest that matches neither //! this binary's output nor its own output row is not a rendering difference, -//! and stays refused, named. +//! and stays refused, named. A current digest over a mutable row that is still +//! that shipped rendering is the same admission: the write that stamped the +//! digest did not finish. A row that matches neither rendering is tamper and +//! stays refused. //! //! Convergence has two outcomes because rendering does. Some released //! renderings are content the current LCM privacy sanitizer withholds, a @@ -662,6 +665,60 @@ mod tests { assert_eq!(stored_output(&snapshot, RECORD_ID).await, current); } + /// The repair above is the shipped rendering, not any disagreement under + /// current provenance. A body neither this binary nor a release wrote is + /// tamper: the audit refuses it and does not rewrite the row. + #[tokio::test] + async fn current_provenance_refuses_a_tampered_output_row() { + let directory = TempDir::new().unwrap(); + let runtime = HostAdmissionTestRuntimeV1::profile(directory.path()) + .await + .unwrap(); + seed(&runtime, &observation()).await.unwrap(); + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .expect("registered profile database"); + + let transaction = database + .runtime_database() + .begin_write_transaction("tamper the projected message body") + .await + .unwrap(); + let updated = transaction + .execute( + "UPDATE session_messages SET text = 'tampered projection body' + WHERE provider = 'codex' AND message_id = ?1", + tracedecay_runtime_core::params![RECORD_ID], + ) + .await + .expect("tamper projected message"); + assert_eq!(updated, 1); + transaction + .execute("DELETE FROM authority_audit_checkpoints", ()) + .await + .unwrap(); + transaction.commit().await.unwrap(); + + let error = + super::super::ensure_authority_invariants(database.runtime_database(), true, false) + .await + .expect_err( + "a tampered projected message under current provenance must stay refused", + ); + let message = error.to_string(); + assert!( + message.contains("projection output rows disagree with deterministic output"), + "{message}" + ); + + let snapshot = database.read_snapshot().await.unwrap(); + assert_eq!( + stored_output(&snapshot, RECORD_ID).await.text, + "tampered projection body", + "refusal must not rewrite the tampered row" + ); + } + #[tokio::test] async fn current_provenance_restores_its_missing_session_row() { let directory = TempDir::new().unwrap(); diff --git a/crates/tracedecay-store/src/canonical_projection.rs b/crates/tracedecay-store/src/canonical_projection.rs index 6cb7ed3280..0ee649de97 100644 --- a/crates/tracedecay-store/src/canonical_projection.rs +++ b/crates/tracedecay-store/src/canonical_projection.rs @@ -10,16 +10,75 @@ use tracedecay_domain::{ use crate::cursor_dispatch::{cursor_dispatch_model, dispatch_text, is_subagent_dispatch_tool}; use crate::provider_descriptor::{ - provider_message_semantics, synthesizes_native_record_id, tool_metadata_normalizer, + ProviderMessageSemantics, provider_message_semantics, synthesizes_native_record_id, + tool_metadata_normalizer, }; use crate::{ ObservationProjection, ProjectionSkipReason, ProjectionStoreError, ProjectionStoreResult, SessionMessageRecord, SessionRecord, WorkflowFactRecord, }; -#[hotpath::measure(label = "store.projection.derive_canonical")] +/// Which projector rendering to derive. +/// +/// Releases through v0.1.0-beta.37 wrote `ShippedRelease`. The reducer is +/// otherwise unchanged; only Codex goal-context semantics were added after +/// that tag. +#[derive(Clone, Copy, Eq, PartialEq)] +enum CanonicalRendering { + Current, + ShippedRelease, +} + +/// Codex goal-context semantics are the only post-release rendering. A shipped +/// derivation withholds them. Every other field is this reducer. +fn rendering_message_semantics( + rendering: CanonicalRendering, + provider: &str, + native_record_kind: &str, + role: &str, + content: &serde_json::Value, + has_native_item_identity: bool, +) -> Option { + match rendering { + CanonicalRendering::ShippedRelease => None, + CanonicalRendering::Current => provider_message_semantics( + provider, + native_record_kind, + role, + content, + has_native_item_identity, + ), + } +} + pub fn derive_canonical_projection( observation: &DurableObservationV1, +) -> ProjectionStoreResult { + derive_canonical_projection_for(observation, CanonicalRendering::Current) +} + +/// Whether `stored` is the message row a shipped release wrote for `observation`. +/// +/// A current-provenance row that still holds that rendering is an interrupted +/// write. Any other body, including a derivation that does not complete, is not. +pub fn stored_message_is_shipped_release_rendering( + observation: &DurableObservationV1, + stored: &SessionMessageRecord, +) -> bool { + let Ok(released) = + derive_canonical_projection_for(observation, CanonicalRendering::ShippedRelease) + else { + return false; + }; + released + .messages() + .any(|projection| projection.message() == stored) +} + +#[hotpath::measure(label = "store.projection.derive_canonical")] +fn derive_canonical_projection_for( + observation: &DurableObservationV1, + rendering: CanonicalRendering, ) -> ProjectionStoreResult { let envelope = CanonicalObservationEnvelopeV1::deserialize(observation.payload()).map_err(|_| { @@ -42,7 +101,7 @@ pub fn derive_canonical_projection( )); } - let mut projected = canonical_message_fields(&envelope)?; + let mut projected = canonical_message_fields_for(rendering, &envelope)?; let session_fields = if envelope.provider().as_str() == "claude" { None } else { @@ -115,7 +174,8 @@ pub fn derive_canonical_projection( let ordinal = i64::try_from(ordinal).map_err(|_| { ProjectionStoreError::Contract(ObservationContractError::InvalidCanonicalPayload) })?; - let metadata_json = canonical_message_metadata( + let metadata_json = canonical_message_metadata_for( + rendering, &envelope, (!session_metadata.is_empty()).then_some(&session_metadata), )?; @@ -310,9 +370,18 @@ fn canonical_session_metadata( serialize_metadata_map(&canonical_session_metadata_map(provider, session)) } +#[cfg(test)] fn canonical_message_metadata( envelope: &CanonicalObservationEnvelopeV1, session_metadata: Option<&serde_json::Map>, +) -> ProjectionStoreResult { + canonical_message_metadata_for(CanonicalRendering::Current, envelope, session_metadata) +} + +fn canonical_message_metadata_for( + rendering: CanonicalRendering, + envelope: &CanonicalObservationEnvelopeV1, + session_metadata: Option<&serde_json::Map>, ) -> ProjectionStoreResult { let serde_json::Value::Object(mut metadata) = serde_json::to_value(envelope) .map_err(|_| ProjectionStoreError::Contract(ObservationContractError::CanonicalEncoding))? @@ -333,7 +402,8 @@ fn canonical_message_metadata( .facts() .iter() .find(|fact| matches!(fact, CanonicalObservationFactV1::Message { .. })) - && let Some(semantics) = provider_message_semantics( + && let Some(semantics) = rendering_message_semantics( + rendering, envelope.provider().as_str(), envelope.native_record_kind(), canonical_role(*role), @@ -687,8 +757,16 @@ fn canonical_cursor_compatibility_message_fields( Ok((primary_message_id, derived)) } +#[cfg(test)] fn canonical_message_fields( envelope: &CanonicalObservationEnvelopeV1, +) -> ProjectionStoreResult> { + canonical_message_fields_for(CanonicalRendering::Current, envelope) +} + +fn canonical_message_fields_for( + rendering: CanonicalRendering, + envelope: &CanonicalObservationEnvelopeV1, ) -> ProjectionStoreResult> { let facts = envelope.facts(); let tool_names = facts @@ -711,7 +789,8 @@ fn canonical_message_fields( { let role = canonical_role(*role); let text = canonical_fact_text(content)?; - if let Some(semantics) = provider_message_semantics( + if let Some(semantics) = rendering_message_semantics( + rendering, envelope.provider().as_str(), envelope.native_record_kind(), role, diff --git a/crates/tracedecay-store/src/lib.rs b/crates/tracedecay-store/src/lib.rs index c09436af3b..7d85f4f7c1 100644 --- a/crates/tracedecay-store/src/lib.rs +++ b/crates/tracedecay-store/src/lib.rs @@ -33,7 +33,8 @@ pub mod session; pub mod transcript; pub use canonical_projection::{ - canonical_fact_text, derive_canonical_projection, workflow_semantic_kind, + canonical_fact_text, derive_canonical_projection, stored_message_is_shipped_release_rendering, + workflow_semantic_kind, }; pub use codex_goal_context::{ CodexGoalContext, CodexGoalContextCorrelation, CodexGoalContextSource, diff --git a/crates/tracedecay/tests/session_suite/observation_projection/failure_audit.rs b/crates/tracedecay/tests/session_suite/observation_projection/failure_audit.rs index ea006aff50..c4b62d5177 100644 --- a/crates/tracedecay/tests/session_suite/observation_projection/failure_audit.rs +++ b/crates/tracedecay/tests/session_suite/observation_projection/failure_audit.rs @@ -767,15 +767,15 @@ async fn authority_reopen_accepts_historical_generation_after_supersession() { ); } -/// A projected message row is derived state, not authority: the immutable -/// observation plus its uniquely owned current provenance re-derive it exactly. -/// Since #1775 (`c55058a3ac`) the reopen audit therefore repairs a diverged -/// output row through the released-rendering convergence ledger instead of -/// degrading the profile forever. Provenance identity, digests that match -/// neither the current nor the stored output, foreign ownership, and -/// conflicting session fields remain hard failures. +/// A projected message body that matches neither this binary's rendering nor +/// the rendering a shipped release wrote is tamper. Profile reopen must name +/// that disagreement and leave the row untouched. An interrupted write whose +/// row is still the shipped rendering is a different admission and is not this +/// case. Provenance identity, digests that match neither the current nor the +/// stored output, foreign ownership, missing rows, and conflicting session +/// fields remain hard failures as well. #[tokio::test] -async fn projected_message_update_is_repaired_on_reopen() { +async fn projected_message_update_invalidates_audit_and_fails_reopen() { let tmp = audited_projection_fixture("session-audit-update", "message-audit-update").await; let runtime = profile_runtime(&tmp).await; let database_path = runtime @@ -793,19 +793,25 @@ async fn projected_message_update_is_repaired_on_reopen() { .unwrap(); drop(raw_conn); - let reopened = HostAdmissionTestRuntimeV1::profile(tmp.path().join(".tracedecay")) - .await - .expect("a diverged output row must be repaired, not refused"); - drop(reopened); + let Err(error) = HostAdmissionTestRuntimeV1::profile(tmp.path().join(".tracedecay")).await + else { + panic!("a tampered projected message must fail profile reopen"); + }; + let message = error.to_string(); + assert!( + message.contains("projection output rows disagree with deterministic output"), + "{message}" + ); assert!( - projected_message_texts(&tmp).await[0].contains("audited projection body"), - "reopen accepted the tampered body instead of re-projecting it" + projected_message_texts(&tmp).await[0].contains("tampered projection body"), + "profile reopen rewrote the tampered body instead of refusing it" ); } -/// The repair above covers a diverged row, never a vanished one: nothing in the -/// convergence ledger inserts a missing message row, so a store whose projected -/// output disappeared still has to be named rather than silently admitted. +/// A vanished projected output is the same hard failure: the convergence +/// ledger rewrites a shipped rendering it can see, and never inserts a missing +/// message row, so a store whose projected output disappeared still has to be +/// named rather than silently admitted. #[tokio::test] async fn projected_message_delete_invalidates_audit_and_fails_reopen() { let tmp = audited_projection_fixture("session-audit-delete", "message-audit-delete").await; From 9b54a5a59f090e2ad0167ae364e83f525297946b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:53:14 +0000 Subject: [PATCH 30/84] fix(sessions): wake temporal refresh from hook ingest Hook completion dropped the execution server's refresh wake whenever a workspace route had selected that server, so committed transcript effects stayed unprojected until an unrelated wake. Codex terminal ingest also wrote after its acknowledgement returned. Both paths now wake the owner of the store that received the write. Co-authored-by: Zack Jackson --- .../src/handlers/hook_runtime/mod.rs | 3 + .../src/handlers/hook_runtime/terminal.rs | 23 +++- .../src/server/live_transcript_refresh.rs | 114 ++++++++++++++++-- crates/tracedecay/src/daemon/projectless.rs | 6 +- crates/tracedecay/src/mcp/server/requests.rs | 5 +- .../session_runtime/temporal_refresh.rs | 30 +++++ 6 files changed, 163 insertions(+), 18 deletions(-) diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/mod.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/mod.rs index 83c2a4c0e4..d6a466b51e 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/mod.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/mod.rs @@ -8,6 +8,7 @@ use tracedecay_global_db::RegisteredGlobalDb; use tracedecay_host_admission::SharedHostAdmissionBroker; use tracedecay_project::project::TraceDecay; use tracedecay_sessions::admission::HostAdmissionOutcome; +use tracedecay_sessions::serving::SessionRefreshWorkerPort; use tracedecay_store_runtime::DaemonSessionRuntimeRegistryV1; use crate::handlers::SessionAuthorities; @@ -155,6 +156,7 @@ pub async fn handle_projectless_hook_runtime( global_db: &RegisteredGlobalDb, session_authorities: SessionAuthorities<'_>, host_admission_broker: std::result::Result<&SharedHostAdmissionBroker, HostAdmissionOutcome>, + user_refresh: Arc, ) -> Result { let action = required_str(&args, "action")?; if !projectless_action_allowed(action, &args) { @@ -183,6 +185,7 @@ pub async fn handle_projectless_hook_runtime( profile_root, &session_runtime_registry, &session_authorities, + Arc::clone(&user_refresh), )?, "hermes_receipt" => { let host_admission_broker = diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/terminal.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/terminal.rs index cc66dd08f5..24ea5b86af 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/terminal.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/terminal.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use std::time::Duration; use tracedecay_automation_runtime::automation::config_error; use tracedecay_domain::errors::Result; +use tracedecay_sessions::serving::SessionRefreshWorkerPort; use super::hermes::user_review; use super::ingest::ingest_transcript_with_cancellation; @@ -20,6 +21,7 @@ pub(super) fn retain_codex_stop( profile_root: &Path, session_runtime_registry: &Arc, session_authorities: &SessionAuthorities<'_>, + user_refresh: Arc, ) -> Result { let session_id = required_str(args, "session_id")?.to_owned(); let user_sessions = session_authorities @@ -51,7 +53,7 @@ pub(super) fn retain_codex_stop( "action": "ingest_transcript", "provider": "codex", "user_scope": true, - "session_id": task_session_id, + "session_id": task_session_id.clone(), }); let authorities = SessionAuthorities::new(None, Some(&user_sessions)) .with_profile_identity(Some(std::sync::Arc::clone(&profile_identity))) @@ -65,10 +67,21 @@ pub(super) fn retain_codex_stop( authorities, &cancellation, ) - .await - .ok() - .and_then(|result| result.get("messages_upserted").and_then(Value::as_u64)) - .is_some_and(|count| count > 0); + .await; + // The parent `codex_stop` acknowledgement returns before + // this task writes. Wake the profile scheduler that owns + // the user store, or the new effects sit until some other + // pass happens to run. + if !user_refresh.wake() { + tracing::warn!( + session_id = %task_session_id, + "retained hook ingest did not wake session temporal refresh" + ); + } + let ingested = ingested + .ok() + .and_then(|result| result.get("messages_upserted").and_then(Value::as_u64)) + .is_some_and(|count| count > 0); if ingested && !cancellation.is_cancelled() && let Some(session_id) = ingest_args.get("session_id").cloned() diff --git a/crates/tracedecay-mcp/src/server/live_transcript_refresh.rs b/crates/tracedecay-mcp/src/server/live_transcript_refresh.rs index 8f1d3b4e74..242d2aae05 100644 --- a/crates/tracedecay-mcp/src/server/live_transcript_refresh.rs +++ b/crates/tracedecay-mcp/src/server/live_transcript_refresh.rs @@ -62,10 +62,15 @@ fn refresh_unavailable(tool_name: &str) -> TraceDecayError { } } +/// Joins the refresh owner of the store this call wrote. +/// +/// `project_wake` and `user_wake` are the execution server's owners. A +/// selected project is that server, so its wake is the project owner, not a +/// reason to ignore it. A missing owner is unavailable. There is no fallback +/// onto the other scope or onto some other project's scheduler. pub async fn join_required_live_transcript_refresh( tool_name: &str, arguments: &Value, - selected_project_owner: bool, project_wake: Option<&dyn SessionRefreshWorkerPort>, user_wake: Option<&dyn SessionRefreshWorkerPort>, ) -> Result { @@ -73,9 +78,8 @@ pub async fn join_required_live_transcript_refresh( return Ok(LiveTranscriptRefreshJoin::NotRequired); }; let wake = match scope { - LiveTranscriptRefreshScope::Project if !selected_project_owner => project_wake, + LiveTranscriptRefreshScope::Project => project_wake, LiveTranscriptRefreshScope::User => user_wake, - LiveTranscriptRefreshScope::Project => None, } .ok_or_else(|| refresh_unavailable(tool_name))?; if wake @@ -90,16 +94,73 @@ pub async fn join_required_live_transcript_refresh( #[cfg(test)] mod tests { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::time::Duration; + use serde_json::json; + use tracedecay_contracts::{SessionTemporalRefreshWakeFuture, SessionTemporalRefreshWakePort}; + use tracedecay_sessions::serving::{ + SessionProjectionServingState, SessionProjectionServingStatus, + SessionProjectionServingStatusPort, + }; + + use super::LiveTranscriptRefreshJoin; use tracedecay_contracts::UnavailableSessionTemporalRefreshWake; + /// Refresh owner whose publication is the state after `wake_and_wait`. + struct PublishingRefresh { + published: AtomicBool, + } + + impl PublishingRefresh { + fn idle() -> Self { + Self { + published: AtomicBool::new(false), + } + } + + fn published(&self) -> bool { + self.published.load(Ordering::Acquire) + } + } + + impl SessionTemporalRefreshWakePort for PublishingRefresh { + fn wake(&self) -> bool { + self.published.store(true, Ordering::Release); + true + } + + fn is_unavailable(&self) -> bool { + false + } + + fn wake_and_wait_until_idle( + &self, + _timeout: Duration, + ) -> SessionTemporalRefreshWakeFuture<'_> { + let published = self.wake(); + Box::pin(async move { published }) + } + } + + impl SessionProjectionServingStatusPort for PublishingRefresh { + fn serving_status(&self) -> SessionProjectionServingStatus { + SessionProjectionServingStatus { + state: SessionProjectionServingState::Current, + last_progress_at_unix_micros: None, + backlog: 0, + blocker: None, + retry_class: None, + } + } + } + #[tokio::test] async fn completed_hook_ingest_fails_when_its_refresh_owner_is_unavailable() { let error = super::join_required_live_transcript_refresh( "tracedecay_hook_runtime", &json!({"action": "ingest_transcript"}), - false, Some(&UnavailableSessionTemporalRefreshWake), None, ) @@ -121,11 +182,11 @@ mod tests { #[tokio::test] async fn user_scope_never_falls_back_to_the_project_refresh_owner() { + let project = PublishingRefresh::idle(); let error = super::join_required_live_transcript_refresh( "tracedecay_hook_runtime", &json!({"action": "ingest_transcript", "user_scope": true}), - false, - Some(&UnavailableSessionTemporalRefreshWake), + Some(&project), None, ) .await @@ -135,24 +196,55 @@ mod tests { error.hook_runtime_context().map(|context| context.0), Some("temporal_refresh_unavailable") ); + assert!( + !project.published(), + "user ingest must not publish through the project refresh owner" + ); } #[tokio::test] - async fn selected_project_never_uses_the_active_projects_refresh_owner() { - let active_project_wake = UnavailableSessionTemporalRefreshWake; + async fn project_ingest_does_not_publish_through_the_user_refresh_owner() { + let user = PublishingRefresh::idle(); let error = super::join_required_live_transcript_refresh( "tracedecay_hook_runtime", &json!({"action": "ingest_transcript"}), - true, - Some(&active_project_wake), None, + Some(&user), ) .await - .expect_err("selected project must require its own refresh owner"); + .expect_err("project ingest must require the project refresh owner"); assert_eq!( error.hook_runtime_context().map(|context| context.0), Some("temporal_refresh_unavailable") ); + assert!( + !user.published(), + "project ingest must not publish through the user refresh owner" + ); + } + + #[tokio::test] + async fn hook_ingest_joins_the_project_refresh_owner() { + let project = PublishingRefresh::idle(); + let user = PublishingRefresh::idle(); + let joined = super::join_required_live_transcript_refresh( + "tracedecay_hook_runtime", + &json!({"action": "ingest_transcript"}), + Some(&project), + Some(&user), + ) + .await + .expect("project hook ingest must join its refresh owner"); + + assert_eq!(joined, LiveTranscriptRefreshJoin::PublicationJoined); + assert!( + project.published(), + "hook ingest must publish through the project refresh owner" + ); + assert!( + !user.published(), + "project hook ingest must not also publish through the user owner" + ); } } diff --git a/crates/tracedecay/src/daemon/projectless.rs b/crates/tracedecay/src/daemon/projectless.rs index 713f1c60d5..85757e6923 100644 --- a/crates/tracedecay/src/daemon/projectless.rs +++ b/crates/tracedecay/src/daemon/projectless.rs @@ -1,6 +1,8 @@ //! Projectless client handling: tool calls served without a mounted project //! (user-scoped LCM, message search, dashboard, doctor, version). +use std::sync::Arc; + use serde_json::json; use tracedecay_daemon_identity::authority; @@ -18,6 +20,7 @@ use tracedecay_mcp::{ }; use tracedecay_session_runtime::session_retrieval::DaemonSessionRetrievalRoot; use tracedecay_sessions::runtime::user_sessions_db_path; +use tracedecay_sessions::serving::SessionRefreshWorkerPort; use tracedecay_store::StoreShardIdV1; use super::*; @@ -558,6 +561,7 @@ async fn projectless_hook_runtime_response( ), ) .await; + let user_refresh: Arc = Arc::new(refresh_wake.clone()); match boxed_projectless_phase( tracedecay_mcp::handlers::hook_runtime::handle_projectless_hook_runtime( arguments.clone(), @@ -572,6 +576,7 @@ async fn projectless_hook_runtime_response( .background_cpu(), ), host_admission_broker, + Arc::clone(&user_refresh), ), ) .await @@ -582,7 +587,6 @@ async fn projectless_hook_runtime_response( Ok(result) => match boxed_projectless_phase(join_required_live_transcript_refresh( "tracedecay_hook_runtime", &arguments, - false, None, Some(&refresh_wake), )) diff --git a/crates/tracedecay/src/mcp/server/requests.rs b/crates/tracedecay/src/mcp/server/requests.rs index 1b4d0b85df..799ca1fbb4 100644 --- a/crates/tracedecay/src/mcp/server/requests.rs +++ b/crates/tracedecay/src/mcp/server/requests.rs @@ -1271,7 +1271,10 @@ impl McpServer { join_required_live_transcript_refresh( &tool_name, &analytics_arguments, - selected_owner.is_some(), + // This server executed the write. Its wakes are + // the owners, including when a workspace route + // selected it. Dropping them leaves the projection + // dirty until an unrelated scheduler wake. self.project_session_refresh_wake.as_deref(), self.user_session_refresh_wake.as_deref(), ), diff --git a/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs b/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs index a551f689cb..5bc875374c 100644 --- a/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs +++ b/crates/tracedecay/tests/session_suite/session_runtime/temporal_refresh.rs @@ -1065,6 +1065,36 @@ impl SessionTemporalRefreshProjector for RecordingDeferredProjector { } } +#[tokio::test] +async fn hook_ingest_join_runs_the_bound_refresh_worker() { + let temp = TempDir::new().unwrap(); + let authority = + registered_test_database(&temp, "hook-ingest-wake", HostAdmissionScope::Profile).await; + let db = authority.database(); + let registry = SessionTemporalRefreshSchedulerRegistry::default(); + let wake = authority.ensure_profile(®istry).await; + let before = registry.profile_pass_count(db.db_path()).await; + + let joined = tracedecay_mcp::server::join_required_live_transcript_refresh( + "tracedecay_hook_runtime", + &json!({"action": "ingest_transcript"}), + Some(&wake), + None, + ) + .await + .expect("hook ingest must join the refresh worker that owns the written store"); + + assert_eq!( + joined, + tracedecay_mcp::server::LiveTranscriptRefreshJoin::PublicationJoined + ); + assert!( + registry.profile_pass_count(db.db_path()).await > before, + "hook ingest must advance the refresh worker, not wait for an unrelated wake" + ); + registry.shutdown().await; +} + #[tokio::test] async fn saturated_recovery_passes_visit_every_operation_before_idling() { let temp = TempDir::new().unwrap(); From 3cb4bc5175b9121987048e12305fd19915044136 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:55:52 +0000 Subject: [PATCH 31/84] fix(hooks): keep ingest commit status off the drain residual The hook and the project catch-up share one per-scope projection queue. Counting observations_committed beside that drain still let the drain residual decide the terminal status: leftover rows promoted a pass that persisted nothing into committed, and an empty residual left a peer-won Codex pass as exact_duplicate. The advisory stop required committed, so the race was the flake. Admission now owns the terminal status for Cursor and Codex. A drain residual cannot commit a pass, hide a commit, or invent a duplicate. Byte-budget deferral is unchanged. The Codex stop accepts the same durable terminals as the Cursor ingest and still rejects accepted_for_replay. Co-authored-by: Zack Jackson --- .../src/handlers/hook_runtime/ingest.rs | 94 ++++++++++++++----- .../handlers/hook_runtime/ingest/kernels.rs | 7 ++ .../src/handlers/hook_runtime/ingest/tests.rs | 90 ++++++++++++++++++ .../src/runtime/hosts/cursor.rs | 53 +++++++---- .../src/runtime/hosts/cursor/tests.rs | 46 +++++++++ .../advisory_runtime_acceptance.rs | 15 ++- 6 files changed, 265 insertions(+), 40 deletions(-) diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest.rs index 846387e8ca..00925df9c0 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest.rs @@ -672,30 +672,31 @@ pub async fn ingest_transcript_with_cancellation( route_admission, observations_committed: route_observations_committed, exact_duplicate: route_exact_duplicate, + admission_owns_commit, } = capture; - // Admission is the durable commit; projection is downstream materialization - // off a queue this scope shares with the project catch-up sweep. Counting - // only the projections this pass drained itself reports a pass whose rows a - // peer drainer took as though it had captured nothing. - let authority_changed = messages_upserted > 0 - || route_observations_committed > 0 - || snapshot_capture + let verdict = ingest_commit_verdict(&IngestCommitAccount { + admission_owns_commit, + observations_committed: route_observations_committed, + route_exact_duplicate, + messages_upserted, + snapshot_messages_upserted: snapshot_capture + .as_ref() + .map_or(0, |capture| capture.stats.messages_upserted), + claude_observations_committed: claude_observation_stats + .as_ref() + .map_or(0, |stats| stats.observations_committed), + claude_cursor_advances: claude_observation_stats .as_ref() - .is_some_and(|capture| capture.stats.messages_upserted > 0) - || claude_observation_stats + .map_or(0, |stats| stats.cursor_advances), + claude_observation_duplicates: claude_observation_stats .as_ref() - .is_some_and(|stats| stats.observations_committed > 0 || stats.cursor_advances > 0); - // A pass that changed nothing is only `accepted_for_replay` when it cannot - // prove the data is already there. Routes that can prove it say so: Claude - // through its duplicate counters, every other route through - // `exact_duplicate`. Without this a replay whose observations a peer - // drainer already projected reports a terminal, non-retryable status that - // neither proves a commit nor invites a retry. - let exact_duplicate = !authority_changed - && (route_exact_duplicate - || claude_observation_stats.as_ref().is_some_and(|stats| { - stats.observation_duplicates > 0 || stats.cursor_duplicates > 0 - })); + .map_or(0, |stats| stats.observation_duplicates), + claude_cursor_duplicates: claude_observation_stats + .as_ref() + .map_or(0, |stats| stats.cursor_duplicates), + }); + let authority_changed = verdict.authority_changed; + let exact_duplicate = verdict.exact_duplicate; let deferred_by_byte_cap = source_deferred || snapshot_capture .as_ref() @@ -779,6 +780,57 @@ pub async fn ingest_transcript_with_cancellation( Ok(output) } +/// The counters a capture route hands the terminal-status assembly. +/// +/// `admission_owns_commit` routes (Cursor, Codex project) already know whether +/// they persisted frames. Their projection drain reads a queue the project +/// catch-up also empties, so `messages_upserted` on those routes is a residual +/// of that queue, not a second copy of the commit. +pub(super) struct IngestCommitAccount { + pub(super) admission_owns_commit: bool, + pub(super) observations_committed: u64, + pub(super) route_exact_duplicate: bool, + pub(super) messages_upserted: u64, + pub(super) snapshot_messages_upserted: u64, + pub(super) claude_observations_committed: u64, + pub(super) claude_cursor_advances: u64, + pub(super) claude_observation_duplicates: u64, + pub(super) claude_cursor_duplicates: u64, +} + +pub(super) struct IngestCommitVerdict { + pub(super) authority_changed: bool, + pub(super) exact_duplicate: bool, +} + +/// Commit status from the route that owns it. +/// +/// When admission owns the commit, a non-zero drain residual cannot promote a +/// pass that persisted nothing into `committed`, and a zero drain cannot hide +/// frames this pass did persist. Routes without an admission tally still read +/// their own message and duplicate counters. +pub(super) fn ingest_commit_verdict(account: &IngestCommitAccount) -> IngestCommitVerdict { + if account.admission_owns_commit { + let authority_changed = account.observations_committed > 0; + return IngestCommitVerdict { + authority_changed, + exact_duplicate: !authority_changed && account.route_exact_duplicate, + }; + } + let authority_changed = account.messages_upserted > 0 + || account.observations_committed > 0 + || account.snapshot_messages_upserted > 0 + || account.claude_observations_committed > 0 + || account.claude_cursor_advances > 0; + IngestCommitVerdict { + authority_changed, + exact_duplicate: !authority_changed + && (account.route_exact_duplicate + || account.claude_observation_duplicates > 0 + || account.claude_cursor_duplicates > 0), + } +} + pub(super) fn complete_ingest_admission( admission: HostAdmissionOutcome, authority_changed: bool, diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest/kernels.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest/kernels.rs index fbf97ef025..245e6f64ba 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest/kernels.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest/kernels.rs @@ -125,6 +125,11 @@ pub(super) struct TranscriptCaptureOutcome { /// `messages_upserted` counts only the projections this pass drained /// itself, which a peer drainer can legitimately take first. pub(super) observations_committed: u64, + /// This route's admission tally is the commit. The projection drain is a + /// shared per-scope queue, so its residual must not enter the terminal + /// status. Routes that have no admission tally leave this false and keep + /// using their own message counts. + pub(super) admission_owns_commit: bool, /// The route committed nothing because its observations were already /// durable. Kept apart from `messages_upserted == 0`, which cannot tell an /// already-committed replay from a pass that captured nothing. @@ -429,6 +434,7 @@ async fn capture_codex_project( source_deferred: admitted.deferred, observations_committed: admitted.observations_committed, exact_duplicate: admitted.exact_duplicate, + admission_owns_commit: true, ..TranscriptCaptureOutcome::default() }) } @@ -457,6 +463,7 @@ fn cursor_capture_outcome( source_deferred: stats.source_deferred, observations_committed: stats.observations_committed, exact_duplicate: stats.exact_duplicate, + admission_owns_commit: true, ..TranscriptCaptureOutcome::default() } } diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest/tests.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest/tests.rs index c62091c5d4..9963dd2df8 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest/tests.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest/tests.rs @@ -1,9 +1,99 @@ use super::super::*; use crate::structured_hook_error_data; use tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1; +use tracedecay_sessions::admission::{HostAdmissionOutcome, HostAdmissionStatus}; use super::*; +fn status_for(account: &IngestCommitAccount) -> HostAdmissionStatus { + let verdict = ingest_commit_verdict(account); + complete_ingest_admission( + HostAdmissionOutcome::accepted_for_replay(), + verdict.authority_changed, + verdict.exact_duplicate, + false, + ) + .status +} + +/// A shared-queue residual is not this pass's commit. Nine projected rows +/// left by a peer, or by another provider on the same scope queue, must stay +/// `accepted_for_replay` when admission persisted nothing and cannot prove a +/// duplicate. +#[test] +fn drain_residual_does_not_commit_an_admission_owned_pass() { + let status = status_for(&IngestCommitAccount { + admission_owns_commit: true, + observations_committed: 0, + route_exact_duplicate: false, + messages_upserted: 9, + snapshot_messages_upserted: 0, + claude_observations_committed: 0, + claude_cursor_advances: 0, + claude_observation_duplicates: 0, + claude_cursor_duplicates: 0, + }); + + assert_eq!(status, HostAdmissionStatus::AcceptedForReplay); +} + +/// The pass persisted two observations and the drain found nothing. The +/// commit still stands. +#[test] +fn admission_commit_stands_when_the_drain_is_empty() { + let status = status_for(&IngestCommitAccount { + admission_owns_commit: true, + observations_committed: 2, + route_exact_duplicate: false, + messages_upserted: 0, + snapshot_messages_upserted: 4, + claude_observations_committed: 1, + claude_cursor_advances: 1, + claude_observation_duplicates: 0, + claude_cursor_duplicates: 0, + }); + + assert_eq!(status, HostAdmissionStatus::Committed); +} + +/// A peer already admitted the source. Residual projected rows must not +/// rewrite that duplicate into a fresh commit. +#[test] +fn drain_residual_does_not_promote_an_exact_duplicate() { + let status = status_for(&IngestCommitAccount { + admission_owns_commit: true, + observations_committed: 0, + route_exact_duplicate: true, + messages_upserted: 3, + snapshot_messages_upserted: 0, + claude_observations_committed: 0, + claude_cursor_advances: 0, + claude_observation_duplicates: 0, + claude_cursor_duplicates: 0, + }); + + assert_eq!(status, HostAdmissionStatus::ExactDuplicate); +} + +/// Hermes and the other routes that have no admission tally still commit +/// from the messages they themselves upserted. +#[test] +fn message_counted_route_still_commits_from_its_own_upserts() { + let status = status_for(&IngestCommitAccount { + admission_owns_commit: false, + observations_committed: 0, + route_exact_duplicate: false, + messages_upserted: 1, + snapshot_messages_upserted: 0, + claude_observations_committed: 0, + claude_cursor_advances: 0, + claude_observation_duplicates: 0, + claude_cursor_duplicates: 0, + }); + + assert_eq!(status, HostAdmissionStatus::Committed); +} + #[test] fn cursor_compaction_response_matches_hook_contract() { let value = cursor_compact_skipped("no messages to compact"); diff --git a/crates/tracedecay-sessions/src/runtime/hosts/cursor.rs b/crates/tracedecay-sessions/src/runtime/hosts/cursor.rs index 4bc895f9b2..16a37a8027 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/cursor.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/cursor.rs @@ -218,6 +218,27 @@ impl CursorSourceAdmissionTally { } } +/// Fold one hook pass's admission over the projection drain it happened to run. +/// +/// The projection queue is per scope. The project catch-up drains it too, so +/// `drain.source_deferred`, `drain.exact_duplicate`, and `drain.messages_upserted` +/// describe whoever last touched that queue, not this pass. A residual there +/// must not hide a commit, invent a duplicate, or turn a byte-finished pass +/// into backpressure. Admission is the commit. +fn account_hook_admission( + mut drain: projection::CursorTranscriptIngestStats, + observations_committed: u64, + fully_replayed: bool, + admission_deferred: bool, + bytes_consumed: u64, +) -> projection::CursorTranscriptIngestStats { + drain.bytes_consumed = bytes_consumed; + drain.source_deferred = admission_deferred; + drain.observations_committed = observations_committed; + drain.exact_duplicate = observations_committed == 0 && fully_replayed; + drain +} + // Cursor JSONL admission chokepoint: the whole per-file admission future is // boxed here so the per-file sweep loop no longer pins each call, keeping the // debug poll frame bounded through the deep ingest recursion chain. @@ -631,19 +652,19 @@ pub async fn try_ingest_cursor_transcript_event_capped_with_admission( admitted.record(&progress); budget.record_progress(progress.bytes_consumed, progress.source_deferred); } - let mut stats = drain_cursor_observation_projections( + let drain = drain_cursor_observation_projections( admission, &scope, &ObservationCancellation::default(), ) .await?; - stats.bytes_consumed = budget.consumed(); - stats.source_deferred |= budget.deferred(); - stats.observations_committed = admitted.observations_committed; - stats.exact_duplicate |= stats.messages_upserted == 0 - && stats.observations_committed == 0 - && admitted.fully_replayed(); - Ok(stats) + Ok(account_hook_admission( + drain, + admitted.observations_committed, + admitted.fully_replayed(), + budget.deferred(), + budget.consumed(), + )) } pub async fn ingest_cursor_user_transcript_event_capped( @@ -781,19 +802,19 @@ pub async fn try_ingest_cursor_user_transcript_event_capped_with_admission( admitted.record(&progress); budget.record_progress(progress.bytes_consumed, progress.source_deferred); } - let mut stats = drain_cursor_observation_projections( + let drain = drain_cursor_observation_projections( admission, &scope, &ObservationCancellation::default(), ) .await?; - stats.bytes_consumed = budget.consumed(); - stats.source_deferred |= budget.deferred(); - stats.observations_committed = admitted.observations_committed; - stats.exact_duplicate |= stats.messages_upserted == 0 - && stats.observations_committed == 0 - && admitted.fully_replayed(); - Ok(stats) + Ok(account_hook_admission( + drain, + admitted.observations_committed, + admitted.fully_replayed(), + budget.deferred(), + budget.consumed(), + )) } pub(in crate::runtime) fn try_ingest_cursor_project_sweep_capped_with_session_ids< diff --git a/crates/tracedecay-sessions/src/runtime/hosts/cursor/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/cursor/tests.rs index b548c3bc95..e80482175c 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/cursor/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/cursor/tests.rs @@ -509,6 +509,52 @@ async fn replayed_cursor_ingest_reports_an_exact_duplicate_not_a_bare_replay() { ); } +/// The shared projection queue's residual is not this pass. A deferred drain +/// with leftover rows does not defer the pass, invent a duplicate, or hide +/// the frames admission persisted. +#[test] +fn hook_admission_ignores_a_shared_drain_residual() { + let committed = account_hook_admission( + CursorTranscriptIngestStats { + messages_upserted: 9, + source_deferred: true, + exact_duplicate: true, + ..CursorTranscriptIngestStats::default() + }, + 2, + false, + false, + 40, + ); + assert_eq!(committed.observations_committed, 2); + assert_eq!(committed.bytes_consumed, 40); + assert_eq!(committed.messages_upserted, 9); + assert!(!committed.source_deferred); + assert!(!committed.exact_duplicate); + + let replayed = account_hook_admission( + CursorTranscriptIngestStats { + messages_upserted: 4, + source_deferred: true, + exact_duplicate: false, + ..CursorTranscriptIngestStats::default() + }, + 0, + true, + false, + 0, + ); + assert_eq!(replayed.observations_committed, 0); + assert!(replayed.exact_duplicate); + assert!(!replayed.source_deferred); + + let deferred = + account_hook_admission(CursorTranscriptIngestStats::default(), 0, false, true, 8); + assert!(deferred.source_deferred); + assert!(!deferred.exact_duplicate); + assert_eq!(deferred.observations_committed, 0); +} + /// The duplicate verdict is evidence, not a default: a source this pass has /// never opened carries no proof that anything was committed before. #[tokio::test] diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/advisory_runtime_acceptance.rs b/crates/tracedecay/tests/runtime_acceptance_suite/advisory_runtime_acceptance.rs index f397d6df9d..ba9e35e8d4 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/advisory_runtime_acceptance.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/advisory_runtime_acceptance.rs @@ -1115,9 +1115,18 @@ async fn packaged_host_ingest_delivers_a_registered_advisory_cycle() { .expect("registered daemon stop response text"), ) .expect("registered daemon stop payload"); - assert_eq!( - stop_payload["status"], "committed", - "registered daemon stop ingest did not commit: {stop_response}" + // Same durable-terminal contract as the Cursor ingest above. The project + // catch-up can admit the rollout first; the hook then reports + // `exact_duplicate`. A drain residual on the shared projection queue must + // not be what flips that into `committed`, and `accepted_for_replay` + // still proves neither a commit nor a duplicate. + assert!( + matches!( + stop_payload["status"].as_str(), + Some("committed" | "exact_duplicate") + ), + "registered daemon stop ingest did not commit: {stop_response}\ndaemon log:\n{}", + std::fs::read_to_string(&daemon_log).expect("read isolated advisory daemon log"), ); let advisory_args = json!({ From 94bc893ffe75ddec1aa209511ba1921bc84a4121 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:55:53 +0000 Subject: [PATCH 32/84] fix(cli): return completed tool results instead of retrying them The project-open client re-sent every pre-admission AfterDelay result until the tool deadline. LCM describe of unavailable history therefore reconnected every 250 ms for 120 s even though the daemon had already answered. A three-attempt cap shortened that storm and also abandoned a mount that takes longer than three observations. Re-send only the publication-window mounting refusal, identified by RUNTIME_MOUNTING_REASON_CODE. Every other completed result is returned on the first observation. Transport project-open errors still wait to their deadline. Co-authored-by: Zack Jackson --- crates/tracedecay-cli/src/cli/dispatch.rs | 6 +- crates/tracedecay-cli/src/tool_command.rs | 35 +++------- .../tracedecay-cli/src/tool_command/tests.rs | 24 ------- .../tests/core_cli_suite/tool_daemon_test.rs | 68 +++++++++++-------- crates/tracedecay-contracts/src/lib.rs | 9 +-- .../src/result/envelope.rs | 65 ++++++++++++++++-- crates/tracedecay-contracts/src/result/mod.rs | 2 +- .../src/result/problem.rs | 9 +++ .../src/result/problem/tests.rs | 4 +- .../tests/project_admission_tests.rs | 9 ++- .../src/invocation/work.rs | 2 +- crates/tracedecay-mcp/src/handlers/edit.rs | 7 +- crates/tracedecay/src/daemon/core_client.rs | 56 ++++++--------- .../src/daemon/tests/invocation_ownership.rs | 5 +- .../tests/mcp_suite/git_correlation_test.rs | 2 +- tests/tool_sweep_suite/runner.py | 3 +- 16 files changed, 167 insertions(+), 139 deletions(-) diff --git a/crates/tracedecay-cli/src/cli/dispatch.rs b/crates/tracedecay-cli/src/cli/dispatch.rs index 3d81b0779b..7f5fdc4670 100644 --- a/crates/tracedecay-cli/src/cli/dispatch.rs +++ b/crates/tracedecay-cli/src/cli/dispatch.rs @@ -66,13 +66,13 @@ pub async fn resolve_cli_application_surface( execute_application_surface(operation, dispatched, executor).await } -/// Delay before re-sending the same CLI application request when its typed -/// pre-admission problem explicitly directs an after-delay retry. +/// Delay before re-sending the same CLI application request when its completed +/// problem is the publication-window mounting refusal. pub(crate) fn surface_retry_delay(result: &ApplicationSurfaceInvocationResult) -> Option { result .result .as_ref() .err()? .problem - .pre_admission_retry_delay() + .owner_mount_resend_delay() } diff --git a/crates/tracedecay-cli/src/tool_command.rs b/crates/tracedecay-cli/src/tool_command.rs index b5f2e0b20f..e9a28e8add 100644 --- a/crates/tracedecay-cli/src/tool_command.rs +++ b/crates/tracedecay-cli/src/tool_command.rs @@ -121,8 +121,6 @@ const PROFILE_REGISTRY_TOOLS: &[&str] = &[ "tracedecay_project_context", ]; -const MAX_SURFACE_ATTEMPTS: usize = 3; - fn tool_deadline_range_error() -> TraceDecayError { TraceDecayError::Config { message: format!( @@ -429,17 +427,13 @@ fn dispatch_cli_application_surface_inner( let handshake = tracedecay::daemon::handshake_for_current_client(project, None, false, false)?; let client = tracedecay_daemon_identity::invocation_client_for_current(handshake)?; - // A cold daemon answers a retryable pre-admission problem while the - // project open still warms in the background (bounded by the daemon's - // foreground open wait). The compatibility tool path rides that state out - // through its project-open retry loop; the typed surface path must present - // the same transport behavior, so re-send the same request per the - // envelope's own retry directive, bounded by both the CLI deadline and - // three attempts so a persistent refusal remains visible to callers. + // A cold daemon answers the mounting refusal while the project open + // still warms in the background. The compatibility tool path rides + // that state out through its project-open retry loop; the typed + // surface path re-sends only that same refusal, until the CLI + // deadline. Every other completed problem is the answer. let mut next_request = Some(request); - let mut attempts = 0usize; let result = loop { - attempts += 1; let request = match next_request.take() { Some(request) => request, None => parse_application_surface_request(operation, tool_args.clone()).map_err( @@ -486,29 +480,18 @@ fn dispatch_cli_application_surface_inner( message: error.to_string(), }, })?; - let Some(delay) = bounded_surface_retry_delay( - crate::cli::dispatch::surface_retry_delay(&result), - attempts, - deadline, - ) else { + let Some(delay) = crate::cli::dispatch::surface_retry_delay(&result) else { break result; }; + if deadline.saturating_duration_since(Instant::now()) <= delay { + break result; + } tokio::time::sleep(delay).await; }; print_cli_application_surface(result, requested_format == RequestedOutputFormat::Json) }) } -fn bounded_surface_retry_delay( - delay: Option, - attempts: usize, - deadline: Instant, -) -> Option { - let delay = delay?; - (attempts < MAX_SURFACE_ATTEMPTS && deadline.saturating_duration_since(Instant::now()) > delay) - .then_some(delay) -} - fn print_cli_application_surface( result: ApplicationSurfaceInvocationResult, raw_json: bool, diff --git a/crates/tracedecay-cli/src/tool_command/tests.rs b/crates/tracedecay-cli/src/tool_command/tests.rs index a685c75973..dd5d9fadde 100644 --- a/crates/tracedecay-cli/src/tool_command/tests.rs +++ b/crates/tracedecay-cli/src/tool_command/tests.rs @@ -92,30 +92,6 @@ fn application_operations_resolve_by_identity_and_by_cli_spelling() { ); } -#[test] -fn retryable_surface_refusals_stop_at_the_attempt_and_deadline_bounds() { - let delay = Duration::from_millis(10); - let roomy_deadline = Instant::now() + Duration::from_secs(1); - assert_eq!( - bounded_surface_retry_delay(Some(delay), 1, roomy_deadline), - Some(delay) - ); - assert_eq!( - bounded_surface_retry_delay(Some(delay), 2, roomy_deadline), - Some(delay) - ); - assert_eq!( - bounded_surface_retry_delay(Some(delay), 3, roomy_deadline), - None, - "the third typed refusal is surfaced instead of retried" - ); - assert_eq!( - bounded_surface_retry_delay(Some(delay), 1, Instant::now() + delay), - None, - "a retry that cannot complete inside the request deadline is refused" - ); -} - #[test] fn whole_payload_invocation_parses_without_a_tool_definition() { let parsed = parse_whole_payload_invocation_with_stdin( diff --git a/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs index dab98b1f5f..e9b5617f83 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs @@ -15,7 +15,8 @@ use crate::common::{ use serde_json::{Value, json}; use tempfile::TempDir; use tracedecay_contracts::{ - ApplicationProblem, ApplicationProblemEnvelope, RequestId, ResultContractRef, SafeDiagnostic, + ApplicationProblem, ApplicationProblemEnvelope, RUNTIME_MOUNTING_REASON_CODE, RequestId, + ResultContractRef, SafeDiagnostic, }; use tracedecay_domain::UtcMicros; use tracedecay_hooks::{HookEventV2, HookHostV1, HookSpoolConfigV1, HookSpoolV1}; @@ -1937,10 +1938,9 @@ fn spawn_scripted_result_sequence_daemon( } } -/// The MCP tool result the daemon renders for a project route whose retained -/// owner is still mounting behind the core publication: `isError` with the -/// typed pre-admission problem and its after-delay retry directive. -fn mounting_owner_tool_result(retry_after_millis: u64) -> Value { +/// The MCP tool result the daemon renders for a completed pre-admission +/// problem whose retry directive is `after_delay`. +fn retry_directed_tool_result(code: &str, message: &str, retry_after_millis: u64) -> Value { let envelope = ApplicationProblemEnvelope::new( ResultContractRef::new( SchemaId::new("schema.retained.fact_store_add.result").expect("schema id"), @@ -1948,15 +1948,9 @@ fn mounting_owner_tool_result(retry_after_millis: u64) -> Value { ) .expect("result contract"), RequestId::new("request.cli.tool.mounting-owner").expect("request id"), - ApplicationProblem::unavailable( - SafeDiagnostic::new( - "application.surface.unavailable", - "The project runtime for this operation is still mounting", - ) - .expect("diagnostic"), - ), + ApplicationProblem::unavailable(SafeDiagnostic::new(code, message).expect("diagnostic")), ) - .expect("mounting owner envelope") + .expect("retry-directed envelope") .with_retry_after_millis(Some(retry_after_millis)) .expect("retry delay"); json!({ @@ -1969,6 +1963,16 @@ fn mounting_owner_tool_result(retry_after_millis: u64) -> Value { }) } +/// The MCP tool result for a project route whose retained owner is still +/// mounting behind the core publication. +fn mounting_owner_tool_result(retry_after_millis: u64) -> Value { + retry_directed_tool_result( + RUNTIME_MOUNTING_REASON_CODE, + "The project runtime for this operation is still mounting", + retry_after_millis, + ) +} + fn fact_store_add_args() -> String { json!({ "category": "project", @@ -2022,6 +2026,8 @@ fn tool_waits_through_an_after_delay_unavailable_within_its_deadline() { socket_path.clone(), "tracedecay_fact_store_add", vec![ + mounting_owner_tool_result(RETRY_AFTER_MILLIS), + mounting_owner_tool_result(RETRY_AFTER_MILLIS), mounting_owner_tool_result(RETRY_AFTER_MILLIS), mounting_owner_tool_result(RETRY_AFTER_MILLIS), json!({ @@ -2051,25 +2057,24 @@ fn tool_waits_through_an_after_delay_unavailable_within_its_deadline() { "stdout must carry the mounted owner's answer, got:\n{stdout}" ); assert!( - !stdout.contains("application.surface.unavailable"), + !stdout.contains(RUNTIME_MOUNTING_REASON_CODE), "a ridden-out mounting state must not reach the caller, got:\n{stdout}" ); let attempts = std::iter::from_fn(|| daemon.requests.try_recv().ok()).count(); assert_eq!( - attempts, 3, - "the CLI must re-send the same request until the owner answers" + attempts, 5, + "the CLI must re-send the same mounting request until the owner answers" ); assert!( - elapsed >= Duration::from_millis(2 * RETRY_AFTER_MILLIS), + elapsed >= Duration::from_millis(4 * RETRY_AFTER_MILLIS), "each retry must wait the delay the directive names, took {elapsed:?}" ); } -/// A completed typed unavailable is still an answer. After three identical -/// results the CLI returns it even when the caller's deadline is much wider; -/// otherwise a permanent diagnostic reconnects every 250 ms until 120 s. +/// A completed authority unavailable is the daemon's answer. Its `after_delay` +/// directive is for the caller; the CLI must not reconnect on it. #[test] -fn tool_caps_repeated_after_delay_results_before_the_deadline() { +fn tool_returns_a_completed_authority_result_without_resending() { let home = TempDir::new().unwrap(); let project = TempDir::new().unwrap(); let socket_dir = TempDir::new().unwrap(); @@ -2081,7 +2086,11 @@ fn tool_caps_repeated_after_delay_results_before_the_deadline() { let daemon = spawn_scripted_result_sequence_daemon( socket_path.clone(), "tracedecay_fact_store_add", - vec![mounting_owner_tool_result(250)], + vec![retry_directed_tool_result( + "application.retained.authority-unavailable", + "The retained operation authority is unavailable: history is not available", + 250, + )], ); let started = Instant::now(); let output = run_command_with_timeout( @@ -2092,7 +2101,7 @@ fn tool_caps_repeated_after_delay_results_before_the_deadline() { assert!( !output.status.success(), - "an owner that never mounts within the deadline must fail\nstdout:\n{}\nstderr:\n{}", + "a completed authority unavailable must fail typed\nstdout:\n{}\nstderr:\n{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); @@ -2102,8 +2111,8 @@ fn tool_caps_repeated_after_delay_results_before_the_deadline() { }); assert_eq!(printed["isError"], true); assert_eq!( - printed["problem"]["code"], "application.surface.unavailable", - "the daemon's typed state must be surfaced after the result retry cap" + printed["problem"]["code"], "application.retained.authority-unavailable", + "the daemon's completed answer must be surfaced, got:\n{stdout}" ); assert_eq!(printed["problem"]["retry"], "after_delay"); let stderr = String::from_utf8_lossy(&output.stderr); @@ -2112,10 +2121,13 @@ fn tool_caps_repeated_after_delay_results_before_the_deadline() { "the process must fail typed, got:\n{stderr}" ); let attempts = std::iter::from_fn(|| daemon.requests.try_recv().ok()).count(); - assert_eq!(attempts, 3, "completed results have one shared attempt cap"); + assert_eq!( + attempts, 1, + "a completed authority result must not be resent" + ); assert!( - elapsed >= Duration::from_millis(500) && elapsed < Duration::from_secs(5), - "the CLI must honor two retry delays but return well before the 10s deadline, took {elapsed:?}" + elapsed < Duration::from_secs(5), + "the CLI must return the completed answer well before the 10s deadline, took {elapsed:?}" ); } diff --git a/crates/tracedecay-contracts/src/lib.rs b/crates/tracedecay-contracts/src/lib.rs index f3049cc9b8..7f91f67774 100644 --- a/crates/tracedecay-contracts/src/lib.rs +++ b/crates/tracedecay-contracts/src/lib.rs @@ -344,10 +344,11 @@ pub use result::{ EvidenceScore, EvidenceScoreKind, EvidenceScoreValue, FreshnessState, IdempotencyKey, LegalAction, Omission, OmissionReason, OpaqueCursor, OperationBudgetUsage, OperationReceipt, OperationTermination, PageCursor, PageState, PolicyDecisionRef, PreviewId, PreviewResult, - ProblemOwningLayer, ProblemTerminality, ReconciliationState, ResultContractRef, ResumeToken, - RetrievalEvidence, RetrieverContribution, RetrieverContributionState, RetryDirective, - RetryScope, SafeDiagnostic, ScoreId, StreamEvent, StreamEventKind, StreamFrontier, StreamGap, - StreamTermination, StreamValidationError, TemporalState, validate_stream, + ProblemOwningLayer, ProblemTerminality, RUNTIME_MOUNTING_REASON_CODE, ReconciliationState, + ResultContractRef, ResumeToken, RetrievalEvidence, RetrieverContribution, + RetrieverContributionState, RetryDirective, RetryScope, SafeDiagnostic, ScoreId, StreamEvent, + StreamEventKind, StreamFrontier, StreamGap, StreamTermination, StreamValidationError, + TemporalState, validate_stream, }; pub use retained_receipts::{ PreparedRetainedEffect, authority_receipt, effective_memory_deadline, evidence_outcome, diff --git a/crates/tracedecay-contracts/src/result/envelope.rs b/crates/tracedecay-contracts/src/result/envelope.rs index 1499f09160..9377fc40a8 100644 --- a/crates/tracedecay-contracts/src/result/envelope.rs +++ b/crates/tracedecay-contracts/src/result/envelope.rs @@ -14,7 +14,7 @@ use super::{ ApplicationExecutionFailureClassV1, ApplicationProblem, ApplicationProblemKind, ApplicationUnavailableClassV1, CancellationStage, EffectReceipt, EffectResult, EvidenceCoverage, EvidencePacket, LegalAction, PreviewResult, ProblemOwningLayer, - ProblemTerminality, RetryDirective, RetryScope, SafeDiagnostic, + ProblemTerminality, RUNTIME_MOUNTING_REASON_CODE, RetryDirective, RetryScope, SafeDiagnostic, }; pub const APPLICATION_PROBLEM_REVISION: u32 = 1; @@ -591,11 +591,14 @@ impl ApplicationProblemRecord { self.terminality == ProblemTerminality::AdmittedTerminal } - /// The delay this problem directs before the same request may be sent - /// again, when it is a retryable pre-admission state such as a warming or - /// still-mounting authority. Admitted terminals and every other retry - /// directive answer `None`: nothing about the request should be repeated - /// on a timer. + /// The delay this problem's retry directive names, when an agent may send + /// the same request again. Admitted terminals and every other retry + /// directive answer `None`. + /// + /// This is not a transport instruction to loop. A retained authority that + /// is unavailable still carries `after_delay` so the caller can choose to + /// retry; only [`owner_mount_resend_delay`] tells the one-shot client to + /// re-send on its own. pub fn pre_admission_retry_delay(&self) -> Option { (self.retryable && self.retry == RetryDirective::AfterDelay && self.is_pre_admission()) .then_some(self.retry_after_millis) @@ -603,6 +606,20 @@ impl ApplicationProblemRecord { .map(Duration::from_millis) } + /// Delay before the one-shot client re-sends this completed result, or + /// `None` when the result is the answer. + /// + /// Classification keys on [`RUNTIME_MOUNTING_REASON_CODE`]. A publication + /// window that is still registering its owner changes if the same request + /// is sent again. Every other completed problem is returned on the first + /// observation, even when its directive is `after_delay`. + pub fn owner_mount_resend_delay(&self) -> Option { + if self.code != RUNTIME_MOUNTING_REASON_CODE { + return None; + } + self.pre_admission_retry_delay() + } + pub fn source(&self) -> &ApplicationProblem { &self.source } @@ -1002,4 +1019,40 @@ mod tests { serde_json::json!(["contract", "request_id", "problem"]) ); } + + fn retry_directed_record(code: &str, delay_millis: u64) -> ApplicationProblemRecord { + let envelope = ApplicationProblemEnvelope::new( + ResultContractRef::new( + SchemaId::new("schema.test.retry-directed.result").expect("schema id"), + 1, + ) + .expect("result contract"), + RequestId::new("request.test.retry-directed").expect("request id"), + ApplicationProblem::unavailable( + SafeDiagnostic::new(code, "The authority named by this code is not ready") + .expect("diagnostic"), + ), + ) + .expect("retry-directed envelope") + .with_retry_after_millis(Some(delay_millis)) + .expect("retry delay"); + *envelope.problem + } + + #[test] + fn only_a_mounting_refusal_is_resent_by_the_one_shot_client() { + let mounting = retry_directed_record(RUNTIME_MOUNTING_REASON_CODE, 40); + let answered = retry_directed_record("application.retained.authority-unavailable", 40); + + assert_eq!( + mounting.owner_mount_resend_delay(), + Some(Duration::from_millis(40)) + ); + assert_eq!( + answered.pre_admission_retry_delay(), + Some(Duration::from_millis(40)), + "the caller-facing directive still names the delay" + ); + assert_eq!(answered.owner_mount_resend_delay(), None); + } } diff --git a/crates/tracedecay-contracts/src/result/mod.rs b/crates/tracedecay-contracts/src/result/mod.rs index abbdb68e36..d59a83277c 100644 --- a/crates/tracedecay-contracts/src/result/mod.rs +++ b/crates/tracedecay-contracts/src/result/mod.rs @@ -19,7 +19,7 @@ pub use evidence::{ pub use problem::{ ApplicationExecutionFailureClassV1, ApplicationProblem, ApplicationProblemKind, ApplicationUnavailableClassV1, LegalAction, ProblemOwningLayer, ProblemTerminality, - RetryDirective, RetryScope, SafeDiagnostic, + RUNTIME_MOUNTING_REASON_CODE, RetryDirective, RetryScope, SafeDiagnostic, }; pub use receipt::{ CancellationObservation, CancellationStage, EffectId, EffectReceipt, EffectResult, diff --git a/crates/tracedecay-contracts/src/result/problem.rs b/crates/tracedecay-contracts/src/result/problem.rs index bc0ca13646..013d638b55 100644 --- a/crates/tracedecay-contracts/src/result/problem.rs +++ b/crates/tracedecay-contracts/src/result/problem.rs @@ -5,6 +5,15 @@ use tracedecay_domain::errors::TraceDecayError; use super::{CancellationStage, EffectReceipt, EffectTermination}; use crate::error::ApplicationContractError; +/// Diagnostic code for an admitted route whose owner is still registering +/// behind the core publication. +/// +/// The one-shot client re-sends the same request only while this code is the +/// problem: the next observation can be the owner's answer. Every other +/// completed problem, including a retryable authority unavailable, is already +/// the daemon's answer and must not be reconnected. +pub const RUNTIME_MOUNTING_REASON_CODE: &str = "application.runtime.mounting"; + /// Safe adapter-independent retry instruction. Adapters preserve it verbatim. #[derive( Clone, Copy, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq, PartialOrd, Ord, Hash, diff --git a/crates/tracedecay-contracts/src/result/problem/tests.rs b/crates/tracedecay-contracts/src/result/problem/tests.rs index 69ba848bee..a4cfcb21b8 100644 --- a/crates/tracedecay-contracts/src/result/problem/tests.rs +++ b/crates/tracedecay-contracts/src/result/problem/tests.rs @@ -157,7 +157,7 @@ fn application_problem_converts_to_typed_trace_decay_error() { let warming = ApplicationProblem::unavailable( SafeDiagnostic::new( - "application.surface.unavailable", + super::RUNTIME_MOUNTING_REASON_CODE, "The project runtime for this operation is still mounting", ) .expect("fixture diagnostic is valid"), @@ -166,7 +166,7 @@ fn application_problem_converts_to_typed_trace_decay_error() { let (reason_code, retryable, _) = warming_error .project_route_context() .expect("warming stays a project-route error"); - assert_eq!(reason_code, "application.surface.unavailable"); + assert_eq!(reason_code, super::RUNTIME_MOUNTING_REASON_CODE); assert!(retryable); } diff --git a/crates/tracedecay-daemon-service/src/invocation/tests/project_admission_tests.rs b/crates/tracedecay-daemon-service/src/invocation/tests/project_admission_tests.rs index 4c83b5a136..75a01fe439 100644 --- a/crates/tracedecay-daemon-service/src/invocation/tests/project_admission_tests.rs +++ b/crates/tracedecay-daemon-service/src/invocation/tests/project_admission_tests.rs @@ -137,7 +137,7 @@ async fn admitted_storage_status_stays_retryable_while_owners_are_warming() { problem .diagnostic() .map(|diagnostic| diagnostic.code.as_str()), - Some("application.surface.unavailable") + Some(tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE) ); } @@ -227,7 +227,7 @@ async fn storage_status_admits_an_owner_registered_under_a_windows_verbatim_root problem .diagnostic() .map(|diagnostic| diagnostic.code.as_str()), - Some("application.surface.unavailable") + Some(tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE) ); } @@ -300,7 +300,10 @@ async fn retained_request_stays_retryable_while_owners_are_warming() { let diagnostic = problem .diagnostic() .expect("a mounting retained owner carries a diagnostic"); - assert_eq!(diagnostic.code, "application.surface.unavailable"); + assert_eq!( + diagnostic.code, + tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE + ); assert!( !diagnostic .message diff --git a/crates/tracedecay-daemon-service/src/invocation/work.rs b/crates/tracedecay-daemon-service/src/invocation/work.rs index 67f7b691f8..7da13757b9 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work.rs @@ -47,7 +47,7 @@ pub(super) fn runtime_mounting_problem(request_id: String) -> DaemonInvocationRe application_problem( request_id, ApplicationProblem::unavailable(SafeDiagnostic { - code: "application.surface.unavailable".to_owned(), + code: tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE.to_owned(), message: "The project runtime for this operation is still mounting".to_owned(), }), ) diff --git a/crates/tracedecay-mcp/src/handlers/edit.rs b/crates/tracedecay-mcp/src/handlers/edit.rs index cee6bfec9b..68ffab992c 100644 --- a/crates/tracedecay-mcp/src/handlers/edit.rs +++ b/crates/tracedecay-mcp/src/handlers/edit.rs @@ -985,7 +985,7 @@ mod tests { let error = source_edit_refusal(DaemonInvocationOutcome::ApplicationProblem { problem: ApplicationProblem::unavailable( SafeDiagnostic::new( - "application.surface.unavailable", + tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE, "The project runtime for this operation is still mounting", ) .unwrap(), @@ -995,7 +995,10 @@ mod tests { let (reason_code, retryable, _) = error .project_route_context() .expect("warming must stay a typed project-route error"); - assert_eq!(reason_code, "application.surface.unavailable"); + assert_eq!( + reason_code, + tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE + ); assert!(retryable); } diff --git a/crates/tracedecay/src/daemon/core_client.rs b/crates/tracedecay/src/daemon/core_client.rs index c85a8df3bb..2368e96c8f 100644 --- a/crates/tracedecay/src/daemon/core_client.rs +++ b/crates/tracedecay/src/daemon/core_client.rs @@ -26,11 +26,6 @@ use super::{ TraceDecayError, error_is_project_open_retryable, }; -/// Completed retryable problem results to observe before returning the typed -/// state to an interactive caller. Transport-level project-open errors are not -/// results and continue to use their explicit deadline. -const MAX_COMPLETED_TOOL_RESULT_ATTEMPTS: usize = 3; - /// Bounded grace a client keeps reading for *after* the caller's request /// deadline has elapsed. /// @@ -495,32 +490,29 @@ fn daemon_tool_call_error(error: JsonRpcError) -> TraceDecayError { } } -/// The delay a completed tool result directs before the same request is sent -/// again, when its typed problem is a retryable pre-admission state. +/// The delay before re-sending a completed tool result, when that result is +/// the publication-window mounting refusal. /// -/// A project-scoped owner that registers behind the core publication (the -/// retained memory authority, the configuration runtime) answers a -/// `RetryDirective::AfterDelay` unavailable while it is still mounting. The -/// daemon renders that record under the tool result's `problem` member, so -/// the one-shot client reads the directive from the same field every MCP -/// client does. An admitted terminal (a partial effect, a permanent owner -/// failure) never directs a delay and is the answer. +/// A project-scoped owner that registers behind the core publication answers +/// `application.runtime.mounting` while it is still mounting. The daemon +/// renders that record under the tool result's `problem` member. An admitted +/// terminal, and every other completed problem (a retained authority that is +/// unavailable, a saturated owner, an observed diagnostic), is the answer: +/// its `after_delay` directive is for the caller, not a transport loop. fn tool_result_retry_after_delay(result: &serde_json::Value) -> Option { let record: tracedecay_contracts::ApplicationProblemRecord = serde_json::from_value(result.get("problem")?.clone()).ok()?; - record.pre_admission_retry_delay() + record.owner_mount_resend_delay() } /// How long to wait before re-sending the request whose outcome is `result`, /// or `None` when that outcome is the answer. /// -/// Two states are ridden out: the daemon's project-open refusal (a JSON-RPC -/// error carrying the warming hint or a saturated open queue) on the client's -/// own cadence, and a completed result whose typed problem directs an -/// after-delay retry, on the delay the directive names. Project-open errors -/// may wait to `deadline`; completed results are also capped by -/// [`MAX_COMPLETED_TOOL_RESULT_ATTEMPTS`] so a persistent authority result is -/// returned instead of hidden behind a reconnect loop. +/// Two states are ridden out to `deadline`: the daemon's project-open refusal +/// (a JSON-RPC error carrying the warming hint or a saturated open queue) on +/// the client's own cadence, and a completed mounting refusal on the delay +/// that result names. Every other completed result is returned on the first +/// observation. fn project_open_retry_wait( result: &Result, deadline: Instant, @@ -547,7 +539,6 @@ async fn call_tool_with_project_open_retry( tool_name: &str, arguments: serde_json::Value, deadline: Instant, - mut completed_result_attempts: usize, ) -> Result { loop { let result = call_tool_within( @@ -558,12 +549,6 @@ async fn call_tool_with_project_open_retry( deadline, ) .await; - if result.is_ok() { - completed_result_attempts = completed_result_attempts.saturating_add(1); - if completed_result_attempts >= MAX_COMPLETED_TOOL_RESULT_ATTEMPTS { - return result; - } - } let Some(wait) = project_open_retry_wait(&result, deadline) else { return result; }; @@ -577,8 +562,9 @@ async fn call_tool_with_project_open_retry( /// accepting daemon. The request deadline travels on the wire; the local read /// waits that deadline plus the 30s response grace. A warming project, or an /// owner still mounting behind its core publication, still retries for at -/// most the 15s open grace, never past this envelope. Callers that need a -/// different budget use [`call_default_tool_within`] or +/// most the 15s open grace, never past this envelope. A completed result that +/// is not that mounting refusal is returned on the first observation. +/// Callers that need a different budget use [`call_default_tool_within`] or /// [`call_default_tool_awaiting_project_open`]. pub async fn call_default_tool( handshake: &DaemonHandshake, @@ -606,7 +592,6 @@ pub async fn call_default_tool( tool_name, arguments, retry_deadline, - usize::from(result.is_ok()), ) .await } @@ -630,8 +615,8 @@ pub async fn call_default_tool_within( /// Bootstrap callers deliberately trigger the cold open they are waiting for, /// so a transport-level warming hint is progress rather than an answer: /// `tracedecay init` asks for a status it can only get after the open completes. -/// Completed application problems are different: after three identical -/// results, `tracedecay tool` returns that typed state for the caller to decide. +/// A completed mounting refusal is the same kind of progress and is re-sent +/// until `deadline`. Every other completed result is returned immediately. pub async fn call_default_tool_awaiting_project_open( handshake: &DaemonHandshake, tool_name: &str, @@ -639,8 +624,7 @@ pub async fn call_default_tool_awaiting_project_open( deadline: Instant, ) -> Result { let socket_path = default_available_socket_path()?; - call_tool_with_project_open_retry(&socket_path, handshake, tool_name, arguments, deadline, 0) - .await + call_tool_with_project_open_retry(&socket_path, handshake, tool_name, arguments, deadline).await } /// Extracts the single JSON payload from an MCP tool result while ignoring diff --git a/crates/tracedecay/src/daemon/tests/invocation_ownership.rs b/crates/tracedecay/src/daemon/tests/invocation_ownership.rs index 1e0480a268..bb453ae7d7 100644 --- a/crates/tracedecay/src/daemon/tests/invocation_ownership.rs +++ b/crates/tracedecay/src/daemon/tests/invocation_ownership.rs @@ -421,7 +421,10 @@ async fn retained_invocation_while_owners_mount_is_retryable_not_unmounted() { let diagnostic = problem .diagnostic() .expect("a mounting retained owner carries a diagnostic"); - assert_eq!(diagnostic.code, "application.surface.unavailable"); + assert_eq!( + diagnostic.code, + tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE + ); assert!( !diagnostic .message diff --git a/crates/tracedecay/tests/mcp_suite/git_correlation_test.rs b/crates/tracedecay/tests/mcp_suite/git_correlation_test.rs index e6cb2364f1..81cfbc336e 100644 --- a/crates/tracedecay/tests/mcp_suite/git_correlation_test.rs +++ b/crates/tracedecay/tests/mcp_suite/git_correlation_test.rs @@ -128,7 +128,7 @@ async fn call(server: &McpServer, tool: &str, mut args: Value) -> Value { .unwrap_or_else(|e| panic!("{tool} should succeed: {e}")); let envelope = extract_json(&result); if envelope.pointer("/problem/code").and_then(Value::as_str) - == Some("application.surface.unavailable") + == Some(tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE) { tokio::time::sleep(std::time::Duration::from_millis(100)).await; continue; diff --git a/tests/tool_sweep_suite/runner.py b/tests/tool_sweep_suite/runner.py index ec1b773f85..bb811bae15 100644 --- a/tests/tool_sweep_suite/runner.py +++ b/tests/tool_sweep_suite/runner.py @@ -716,8 +716,9 @@ def _mounting_producer_call( f"{tool} producer omitted the enabled _meta.duration_us receipt" ) return response + # Must match tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE. if ( - row["problem_code"] != "application.surface.unavailable" + row["problem_code"] != "application.runtime.mounting" or time.monotonic() >= ends_at ): raise SweepError( From 5d0c11139b6174385e7ca5a6e6230c8c41b94069 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:58:20 +0000 Subject: [PATCH 33/84] fix(global-db): rewrite stale raw twins under current provenance The output digest covers the message, not its LCM raw twin, and the audit checkpoint only watched session_messages. A twin could drift under a current digest and a trusted checkpoint would never see it. The ingest upsert also refused a twin whose session_id had drifted, so the message repair rolled back. Invalidate the checkpoint when a provenance-owned twin changes, and rewrite a twin that is not what a fresh projection write stores, including adopting a drifted session. An unverifiable twin is not a protected rendering. Co-authored-by: Zack Jackson --- .../src/observation_projection/apply.rs | 32 +++++ .../src/observation_projection/state.rs | 84 ++++++++++- .../src/schema_contract/invariants/audit.rs | 49 ++++++- .../invariants/released_rendering.rs | 134 ++++++++++++++++++ .../schema_contract/invariants/triggers.rs | 44 ++++++ .../tracedecay-global-db/src/schema_stages.rs | 30 ++-- 6 files changed, 353 insertions(+), 20 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation_projection/apply.rs b/crates/tracedecay-global-db/src/observation_projection/apply.rs index a9ad8c7ce1..58026795d8 100644 --- a/crates/tracedecay-global-db/src/observation_projection/apply.rs +++ b/crates/tracedecay-global-db/src/observation_projection/apply.rs @@ -640,6 +640,34 @@ pub(super) async fn apply_session( /// A deterministic sanitization refusal keeps its typed class: mapping it to /// `Storage` would schedule an endless environmental retry for content that /// can never succeed, permanently poisoning the sequential projection queue. +/// Aligns a provenance-owned raw twin onto the projection's session before +/// the content upsert. +/// +/// The ingest upsert refuses a row whose `session_id` differs, so a drifted +/// twin blocks the rewrite that uniquely owned current provenance authorizes. +/// `(provider, message_id)` is that ownership key; `session_id` is a field of +/// the twin, not a second owner. Callers reach this only after that ownership +/// is already proven (an existing projected message, or released-rendering +/// convergence). A first insert of an unowned identity must not adopt a +/// foreign twin and does not call this. +async fn adopt_owned_projection_raw_session( + conn: &impl Executor, + message: &SessionMessageRecord, +) -> ProjectionStoreResult<()> { + conn.execute( + "UPDATE lcm_raw_messages SET session_id = ?3 + WHERE provider = ?1 AND message_id = ?2 AND session_id <> ?3", + params![ + message.provider.as_str(), + message.message_id.as_str(), + message.session_id.as_str(), + ], + ) + .await + .map(|_| ()) + .map_err(|error| storage("adopt projection raw session", error)) +} + async fn upsert_projected_raw_message( conn: &impl Executor, message: &SessionMessageRecord, @@ -844,6 +872,7 @@ pub(in super::super) async fn converge_released_output_rendering( let message = projection.message(); supersede_projected_message(conn, message).await?; if message.provider != "hermes" { + adopt_owned_projection_raw_session(conn, message).await?; match upsert_projected_raw_message(conn, message).await { Ok(()) => {} Err(ProjectionStoreError::SanitizationRefused { @@ -1022,6 +1051,9 @@ async fn apply_rows( } }; if projected_message.provider != "hermes" && !preserve_protected_payload { + if existing.is_some() { + adopt_owned_projection_raw_session(conn, projected_message).await?; + } upsert_projected_raw_message(conn, projected_message).await?; } Ok(transition == MessageTransition::Insert) diff --git a/crates/tracedecay-global-db/src/observation_projection/state.rs b/crates/tracedecay-global-db/src/observation_projection/state.rs index c3c911e745..f3678fde55 100644 --- a/crates/tracedecay-global-db/src/observation_projection/state.rs +++ b/crates/tracedecay-global-db/src/observation_projection/state.rs @@ -789,9 +789,19 @@ pub(in super::super) struct ProjectionOutputAuthority { pub(in super::super) canonical: DurableObservationV1, } +/// The LCM raw twin stored beside one projected message. Not part of the +/// output digest; current provenance still authorizes it because the twin is +/// derived from the same observation. +pub(in super::super) struct ProjectionRawTwin { + pub(in super::super) session_id: String, + pub(in super::super) storage_kind: String, + pub(in super::super) content: String, +} + pub(in super::super) struct ProjectionRowsBatch { sessions: HashMap<(String, String), SessionRecord>, messages: HashMap<(String, String), SessionMessageRecord>, + raw_twins: HashMap<(String, String), ProjectionRawTwin>, } impl ProjectionRowsBatch { @@ -812,6 +822,15 @@ impl ProjectionRowsBatch { self.messages .get(&(provider.to_owned(), message_id.to_owned())) } + + pub(in super::super) fn raw_twin( + &self, + provider: &str, + message_id: &str, + ) -> Option<&ProjectionRawTwin> { + self.raw_twins + .get(&(provider.to_owned(), message_id.to_owned())) + } } pub(in super::super) async fn read_projection_rows_batch( @@ -819,6 +838,7 @@ pub(in super::super) async fn read_projection_rows_batch( outputs: &BTreeSet<(String, String)>, ) -> ProjectionStoreResult { let mut messages = HashMap::with_capacity(outputs.len()); + let mut raw_twins = HashMap::with_capacity(outputs.len()); let requested_keys = outputs.iter().collect::>(); for chunk in requested_keys.chunks(OUTPUT_AUTHORITY_BATCH_KEYS) { let requested = serde_json::to_string( @@ -875,6 +895,45 @@ pub(in super::super) async fn read_projection_rows_batch( message, ); } + drop(rows); + let mut rows = conn + .query( + "SELECT raw.provider, raw.message_id, raw.session_id, raw.storage_kind, + COALESCE(raw.content, '') + FROM json_each(?1) AS requested + CROSS JOIN lcm_raw_messages AS raw + WHERE raw.provider = json_extract(requested.value, '$.provider') + AND raw.message_id = json_extract(requested.value, '$.message_id')", + params![requested.as_str()], + ) + .await + .map_err(|error| storage("read projected raw twins", error))?; + while let Some(row) = rows + .next() + .await + .map_err(|error| storage("read projected raw twins", error))? + { + let provider = row + .get::(0) + .map_err(|error| storage("decode projected raw twins", error))?; + let message_id = row + .get::(1) + .map_err(|error| storage("decode projected raw twins", error))?; + raw_twins.insert( + (provider, message_id), + ProjectionRawTwin { + session_id: row + .get(2) + .map_err(|error| storage("decode projected raw twins", error))?, + storage_kind: row + .get(3) + .map_err(|error| storage("decode projected raw twins", error))?, + content: row + .get(4) + .map_err(|error| storage("decode projected raw twins", error))?, + }, + ); + } } let session_keys = messages @@ -945,7 +1004,11 @@ pub(in super::super) async fn read_projection_rows_batch( } } - Ok(ProjectionRowsBatch { sessions, messages }) + Ok(ProjectionRowsBatch { + sessions, + messages, + raw_twins, + }) } /// The batched ownership resolution behind [`read_output_authorities`]. @@ -1322,10 +1385,21 @@ pub(super) async fn protected_message_rows_compatible( == Some(expected_hash.as_str()) && payload_ref.is_some_and(|payload_ref| actual.text.contains(payload_ref)); if !external { - let raw = - tracedecay_lcm::schema::load_raw_message(conn, &actual.provider, &actual.message_id) - .await - .map_err(|error| storage("read protected projection output", error))?; + // A twin that fails its own receipt is not a protected rendering of + // this projection. Callers treat that as an ordinary output mismatch + // and, when current provenance uniquely owns the output, rewrite it. + // A database fault is still a fault. + let raw = match tracedecay_lcm::schema::load_raw_message( + conn, + &actual.provider, + &actual.message_id, + ) + .await + { + Ok(raw) => raw, + Err(tracedecay_lcm::LcmError::PayloadIntegrityMismatch) => return Ok(false), + Err(error) => return Err(storage("read protected projection output", error)), + }; let Some(raw) = raw else { return Ok(false); }; diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs index 565c0a309d..d61bd06457 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs @@ -2,6 +2,7 @@ use std::collections::{BTreeSet, HashMap}; use futures_util::future::try_join_all; use tracedecay_domain::DurableObservationV1; +use tracedecay_privacy::sanitize_lcm_payload_text; use tracedecay_store::{ ObservationProjection, ProjectionSkipReason, ProjectionStoreError, SESSION_MESSAGE_PROJECTOR_VERSION, SessionMessageProjection, WorkflowFactProjection, @@ -828,7 +829,22 @@ async fn validate_message_projection_row( .message(&owner_message.provider, &owner_message.message_id) .is_some(); match verify_owner_output_rows(conn, resolved, &owner_projection).await { - Ok(()) => {} + Ok(()) => { + // Message equality is not the whole output. The raw twin is + // derived from the same observation and is not covered by the + // digest, so a matching message can still sit on a stale twin. + // Protected rows are not this arm: their stored message differs + // from the projection, and that compatibility already checked + // the twin. + if resolved + .projection_rows + .message(&owner_message.provider, &owner_message.message_id) + .is_some_and(|stored| stored == owner_message) + && owned_raw_twin_needs_rewrite(&owner_projection, &resolved.projection_rows)? + { + resolved.released.record(&owner_projection); + } + } Err(ProjectionStoreError::OutputCollision { provider, message_id, @@ -890,6 +906,37 @@ async fn verify_owner_output_rows( .await } +/// Whether the LCM raw twin of a message that already matches this projection +/// is not the twin a fresh projection write would store. +/// +/// Hermes projections have no raw twin. A sanitizer quarantine is itself the +/// current rendering, so the caller records the projection for the same +/// converge path a fresh capture uses. A sanitizer fault stays a typed refusal. +fn owned_raw_twin_needs_rewrite( + projection: &SessionMessageProjection, + rows: &ProjectionRowsBatch, +) -> tracedecay_domain::errors::Result { + let message = projection.message(); + if message.provider == "hermes" { + return Ok(false); + } + let expected = match sanitize_lcm_payload_text(&message.text) { + Ok(sanitized) => sanitized.sanitized_text().to_owned(), + Err(error) if error.is_quarantine_verdict() => return Ok(true), + Err(error) => { + return Err(authority_violation(format!( + "projection raw twin sanitizer failed: {error}" + ))); + } + }; + let Some(raw) = rows.raw_twin(&message.provider, &message.message_id) else { + return Ok(true); + }; + Ok(raw.storage_kind != "inline" + || raw.session_id != message.session_id + || raw.content != expected) +} + #[allow(clippy::too_many_arguments)] async fn validate_message_projection( conn: &impl QueryExecutor, diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs index 760c1dcbd9..ee213c10fa 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs @@ -662,6 +662,140 @@ mod tests { assert_eq!(stored_output(&snapshot, RECORD_ID).await, current); } + /// The digest covers the message, not its LCM raw twin. A twin can be + /// rewritten under a still-current message and provenance; reopen has to + /// restore the twin a fresh projection write stores. + #[tokio::test] + async fn current_provenance_repairs_a_stale_raw_twin() { + let directory = TempDir::new().unwrap(); + let runtime = HostAdmissionTestRuntimeV1::profile(directory.path()) + .await + .unwrap(); + seed(&runtime, &observation()).await.unwrap(); + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .expect("registered profile database"); + let snapshot = database.read_snapshot().await.unwrap(); + let current = stored_output(&snapshot, RECORD_ID).await; + drop(snapshot); + + let transaction = database + .runtime_database() + .begin_write_transaction("stale the raw twin under current provenance") + .await + .unwrap(); + let updated = transaction + .execute( + "UPDATE lcm_raw_messages + SET content = 'stale raw body', content_hash = 'stale', + snippet_text = 'stale raw body', index_text = 'stale raw body' + WHERE provider = 'codex' AND message_id = ?1", + tracedecay_runtime_core::params![RECORD_ID], + ) + .await + .expect("stale the raw twin"); + assert_eq!(updated, 1); + transaction.commit().await.unwrap(); + + let snapshot = database.read_snapshot().await.unwrap(); + let stale = stored_output(&snapshot, RECORD_ID).await; + drop(snapshot); + assert_eq!(stale.digest, current.digest); + assert_eq!(stale.text, current.text); + assert_eq!(stale.raw_index_text, "stale raw body"); + + super::super::ensure_authority_invariants(database.runtime_database(), false, false) + .await + .expect("current provenance must repair its stale raw twin"); + + let snapshot = database.read_snapshot().await.unwrap(); + assert_eq!(stored_output(&snapshot, RECORD_ID).await, current); + } + + /// A drifted raw `session_id` is not a second owner of a provenance-bound + /// output. The ingest upsert refuses that row; convergence still has to + /// finish the rewrite or the stale message stays served. + #[tokio::test] + async fn current_provenance_adopts_a_raw_twin_on_a_stale_session() { + let directory = TempDir::new().unwrap(); + let runtime = HostAdmissionTestRuntimeV1::profile(directory.path()) + .await + .unwrap(); + seed(&runtime, &observation()).await.unwrap(); + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .expect("registered profile database"); + let snapshot = database.read_snapshot().await.unwrap(); + let current = stored_output(&snapshot, RECORD_ID).await; + drop(snapshot); + + let transaction = database + .runtime_database() + .begin_write_transaction("move the raw twin off its projection session") + .await + .unwrap(); + transaction + .execute( + "INSERT INTO sessions ( + provider, session_id, project_key, project_path, title, + started_at, ended_at, transcript_path, metadata_json, + parent_session_id, is_subagent, agent_id, parent_tool_use_id + ) + SELECT provider, 'stale-raw-session', project_key, project_path, title, + started_at, ended_at, transcript_path, metadata_json, + parent_session_id, is_subagent, agent_id, parent_tool_use_id + FROM sessions WHERE provider = 'codex' AND session_id = ?1", + tracedecay_runtime_core::params![SESSION], + ) + .await + .expect("create the session the drifted twin points at"); + transaction + .execute( + "UPDATE session_messages SET text = 'stale message body' + WHERE provider = 'codex' AND message_id = ?1", + tracedecay_runtime_core::params![RECORD_ID], + ) + .await + .expect("stale the message row"); + let updated = transaction + .execute( + "UPDATE lcm_raw_messages + SET session_id = 'stale-raw-session', content = 'stale raw body', + content_hash = 'stale', snippet_text = 'stale raw body', + index_text = 'stale raw body' + WHERE provider = 'codex' AND message_id = ?1", + tracedecay_runtime_core::params![RECORD_ID], + ) + .await + .expect("move the raw twin"); + assert_eq!(updated, 1); + transaction.commit().await.unwrap(); + + super::super::ensure_authority_invariants(database.runtime_database(), false, false) + .await + .expect("current provenance must adopt its drifted raw twin"); + + let snapshot = database.read_snapshot().await.unwrap(); + assert_eq!(stored_output(&snapshot, RECORD_ID).await, current); + let mut rows = snapshot + .query( + "SELECT session_id FROM lcm_raw_messages + WHERE provider = 'codex' AND message_id = ?1", + tracedecay_runtime_core::params![RECORD_ID], + ) + .await + .expect("read repaired raw session"); + assert_eq!( + rows.next() + .await + .expect("read repaired raw session") + .expect("raw twin row") + .get::(0) + .unwrap(), + SESSION + ); + } + #[tokio::test] async fn current_provenance_restores_its_missing_session_row() { let directory = TempDir::new().unwrap(); diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/triggers.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/triggers.rs index 7defbbff58..0924d3a151 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/triggers.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/triggers.rs @@ -227,6 +227,38 @@ const PROJECTION_AUDIT_INVALIDATION: &[Trigger] = &[ WHERE audit_name = 'observation-authority'; END", }, + // The message-row triggers above do not see the LCM raw twin. A twin can + // drift (content, session identity) while the message row and the current + // provenance digest stay put, and the trusted checkpoint would then skip + // it forever. Invalidate on the same ownership predicate. + Trigger { + name: "projection_raw_audit_invalidate_update_v1", + table: "lcm_raw_messages", + create_sql: "CREATE TRIGGER projection_raw_audit_invalidate_update_v1 + AFTER UPDATE ON lcm_raw_messages + WHEN EXISTS ( + SELECT 1 FROM observation_projection_provenance + WHERE output_provider = OLD.provider + AND output_message_id = OLD.message_id + ) BEGIN + DELETE FROM authority_audit_checkpoints + WHERE audit_name = 'observation-authority'; + END", + }, + Trigger { + name: "projection_raw_audit_invalidate_delete_v1", + table: "lcm_raw_messages", + create_sql: "CREATE TRIGGER projection_raw_audit_invalidate_delete_v1 + AFTER DELETE ON lcm_raw_messages + WHEN EXISTS ( + SELECT 1 FROM observation_projection_provenance + WHERE output_provider = OLD.provider + AND output_message_id = OLD.message_id + ) BEGIN + DELETE FROM authority_audit_checkpoints + WHERE audit_name = 'observation-authority'; + END", + }, Trigger { name: "projection_checkpoint_audit_invalidate_regression_v1", table: "observation_projection_checkpoints", @@ -1650,6 +1682,15 @@ struct ReleasedV3TriggerDrift { released: &'static str, } +/// Triggers added after the v3 inventory. A released store is admitted on the +/// published bodies, then schema convergence installs these and the missing +/// contract forces the exhaustive repair pass. Requiring them at admission +/// would reset every beta.25–beta.37 profile. +const POST_RELEASED_V3_TRIGGERS: &[&str] = &[ + "projection_raw_audit_invalidate_update_v1", + "projection_raw_audit_invalidate_delete_v1", +]; + const RELEASED_V3_TRIGGER_DRIFT: &[ReleasedV3TriggerDrift] = &[ ReleasedV3TriggerDrift { trigger: "session_refresh_progress_insert_guard_v1", @@ -1721,6 +1762,9 @@ pub async fn released_v3_invariant_triggers_intact( let released = released_v3_trigger_contracts()?; for invariant in INVARIANTS { for trigger in invariant.triggers { + if POST_RELEASED_V3_TRIGGERS.contains(&trigger.name) { + continue; + } let expected = released .iter() .find(|(name, _)| *name == trigger.name) diff --git a/crates/tracedecay-global-db/src/schema_stages.rs b/crates/tracedecay-global-db/src/schema_stages.rs index 34cc15ba64..56aeff0f89 100644 --- a/crates/tracedecay-global-db/src/schema_stages.rs +++ b/crates/tracedecay-global-db/src/schema_stages.rs @@ -848,6 +848,22 @@ async fn install_registered_schema_stage_sequence( .execute_batch(RUNTIME_LEDGER_SCHEMA) .await .map_err(|error| global_db_operation_error("initialize runtime writer ledger", error))?; + // Projection raw-twin triggers sit on `lcm_raw_messages`. The table has to + // exist before those triggers are created, including on a fresh store + // whose authority triggers are installed in this same transaction. + tracedecay_lcm::schema::ensure_lcm_schema_in_transaction(transaction) + .await + .map_err(|error| match error { + tracedecay_lcm::LcmError::ProfileResetRequired { + found_version, + required_version, + } => tracedecay_domain::errors::TraceDecayError::ProfileResetRequired { + component: "LCM", + found_version, + required_version, + }, + error => global_db_operation_error("initialize LCM schema", error), + })?; // `force_exhaustive` means admission observed damaged or missing guard // triggers (for example a dropped guarded table takes its triggers with // it). Reinstall them here so the post-commit contract validation sees a @@ -865,20 +881,6 @@ async fn install_registered_schema_stage_sequence( )); } } - - tracedecay_lcm::schema::ensure_lcm_schema_in_transaction(transaction) - .await - .map_err(|error| match error { - tracedecay_lcm::LcmError::ProfileResetRequired { - found_version, - required_version, - } => tracedecay_domain::errors::TraceDecayError::ProfileResetRequired { - component: "LCM", - found_version, - required_version, - }, - error => global_db_operation_error("initialize LCM schema", error), - })?; tracedecay_sessions::runtime::git_correlation::ensure_git_correlation_receipt_schema_in_transaction( transaction, ) From 151836457716b30cde4c7009f13d081ed2c49022 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 07:00:09 +0000 Subject: [PATCH 34/84] test(retention): assert a vanished census open defers Opening a listed generation that publication already unlinked must not be Storage. The census defers as store-busy; a non-NotFound open stays storage. Co-authored-by: Zack Jackson --- .../src/code_index_generations/tests.rs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs index 116a8becc6..f8e99ab2bf 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs @@ -3107,3 +3107,28 @@ fn recovery_completes_a_committed_rewrite_that_never_reached_the_pointer() { plan_code_generation_retention(fixture.store.path(), &BTreeSet::new()) .expect("a recovered store must stay plannable"); } + +/// The census opens every name `read_dir` just returned. Publication can +/// unlink that name first. `NotFound` is the same deferral as a held writer, +/// not a storage failure. Any other open failure stays storage. +#[test] +fn vanished_listed_generation_open_defers_instead_of_storage_loss() { + let root = tempfile::tempdir().expect("census root"); + let missing = root.path().join(format!("generation-{:064x}.json", 1)); + let error = super::generation_scan::read_generation_format_revision(&missing, &|| false) + .expect_err("a vanished listed generation defers the census"); + assert!( + matches!(error, CodeGenerationRetentionErrorV1::GenerationStoreBusy), + "a missing listed generation is a publisher race, not a storage failure: {error:?}" + ); + + let directory = root.path().join("not-a-generation-file"); + std::fs::create_dir(&directory).expect("directory where a file was listed"); + let storage_error = + super::generation_scan::read_generation_format_revision(&directory, &|| false) + .expect_err("a directory is not a vanished file"); + assert!( + matches!(storage_error, CodeGenerationRetentionErrorV1::Storage(_)), + "non-NotFound census I/O stays a storage failure: {storage_error:?}" + ); +} From 87157426c4f9055bd85d05c7e91c1e54ef8a4ce9 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 07:00:37 +0000 Subject: [PATCH 35/84] test(code-index): drop the redundant idle-admission hold Master's coalesced_publication_failure fix (0d328fa0a9) takes the generation-store lock before corrupting the active pointer, and its own comment records why the background admission permit is not the proof: the racer releases that permit before it finishes attaching the generation's text artifact, so holding it does not mean the store is quiet. Being granted the store lock does. The extra permit hold this branch layered on top adds a second 5s spin for a race the landed fix already closes. Co-Authored-By: Claude Opus 5 (1M context) --- .../flight_tests.rs | 32 ++----------------- 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs index feac73d938..bc1388df7f 100644 --- a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs +++ b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs @@ -2,7 +2,6 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Condvar, Mutex}; use tracedecay_code_index::production::CodeIndexProductionErrorV1; -use tracedecay_code_index_retention::code_index_generations::try_acquire_code_generation_store_lock; use super::*; @@ -158,32 +157,6 @@ fn assert_publication_error(error: CodeIndexSchedulerErrorV1) { ); } -/// Hold the only background permit once no pass is in flight. -/// -/// Text seating keeps `reconcile_in_progress` after it drops the scheduler -/// mutex, and that pass can still rename a valid active pointer. A truncated -/// pointer written in that window is not a closed fault. Occupying the permit -/// while the owner has not entered its pass stops that rewrite. -async fn hold_idle_background_admission( - registry: &CodeIndexSchedulerRegistryV1, -) -> tokio::sync::OwnedSemaphorePermit { - let admission = registry.background_reconcile_admission(); - let deadline = std::time::Instant::now() + Duration::from_secs(5); - loop { - if registry.memory_stats().await.reconciling_worktrees == 0 - && let Ok(permit) = admission.clone().try_acquire_owned() - && registry.memory_stats().await.reconciling_worktrees == 0 - { - return permit; - } - assert!( - std::time::Instant::now() <= deadline, - "background reconcile did not go idle before publication fault injection" - ); - tokio::time::sleep(Duration::from_millis(2)).await; - } -} - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn aborted_flight_owner_wakes_follower_and_allows_a_fresh_owner() { let fixture = fixture(); @@ -269,8 +242,6 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { let registry = Arc::new(mount(fixture.path(), &store, 1).await); let baseline = latest(®istry, fixture.path()).await; let request = request_for(&baseline, "pkg"); - let idle_admission = hold_idle_background_admission(®istry).await; - registry.clear_pending_wake_for_scope(&request.scope).await; let hold = SchedulerHold::acquire(®istry, fixture.path()).await; let (owner_control, owner_entered) = BlockingNthControl::new(4); @@ -326,6 +297,8 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { // and any writer that starts after it is released reads the corruption // under the lock and refuses instead of overwriting it. let pointer_bytes = { + use tracedecay_code_index_retention::code_index_generations::try_acquire_code_generation_store_lock; + let store_lock = tokio::time::timeout(Duration::from_secs(5), async { loop { if let Some(lock) = try_acquire_code_generation_store_lock(&scoped_store) @@ -343,7 +316,6 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { drop(store_lock); pointer_bytes }; - drop(idle_admission); owner_control.release(); hold.release(); From b59e66ec049fcc7a4d07b1c34b74e67ae023339f Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 07:01:13 +0000 Subject: [PATCH 36/84] test(memory): state the real text-build growth budget The reclaim-band test asserted a 4 GiB growth budget from a 22 GiB baseline while its own narrative and RSS samples used 24 GiB, then claimed a 3 GiB replacement fits. Text-artifact admission reduces to `high_watermark - observed`, so at 24 GiB observed the production budget is 2 GiB, not 4. Assert that figure and the 1536 MiB builder floor it clears, so the test stops overstating what memory.max buys. Co-Authored-By: Claude Fable 5.1 --- .../src/resident_memory/tests.rs | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/resident_memory/tests.rs b/crates/tracedecay-runtime-core/src/resident_memory/tests.rs index eeb78ea8b5..292ad8f2cd 100644 --- a/crates/tracedecay-runtime-core/src/resident_memory/tests.rs +++ b/crates/tracedecay-runtime-core/src/resident_memory/tests.rs @@ -226,7 +226,8 @@ fn finite_ancestor_limit_bounds_an_unlimited_process_cgroup() { /// The slice owns `memory.max` and the service owns `memory.high`. /// /// On a 128 GiB host those are 30 GiB and 26 GiB. RSS at 24 GiB is still under -/// the reclaim line, and a 3 GiB replacement fits in the 4 GiB band down to it. +/// the reclaim line, so the authority admits growth instead of latching at a +/// percentage of a ceiling the operator never set. #[test] fn slice_max_and_service_high_keep_the_reclaim_band_usable() { let gib = 1024 * 1024 * 1024; @@ -277,19 +278,25 @@ fn slice_max_and_service_high_keep_the_reclaim_band_usable() { key("project-a", "worktree-a", "generation-a", "text-build"), bytes(3 * gib), ) - .expect("a 3 GiB replacement is admitted at 24 GiB RSS"); - + .expect("the process authority admits a 3 GiB reservation at 24 GiB RSS"); + + // Text-artifact admission spends the band down to the reclaim line, never + // down to memory.max: `text_artifact_admitted_build_budget` subtracts the + // same watermark headroom it charges, so its growth budget reduces to + // `high_watermark - observed`. At 24 GiB observed that is 2 GiB, clearing + // the 1536 MiB builder floor. The 90%-of-26 GiB watermark left 0 and + // deadlocked the replacement build. let headroom = detected .limit_bytes .get() .saturating_sub(pressure.high_watermark_bytes()); - let available = detected + let available_for_growth = detected .limit_bytes .get() - .saturating_sub(22 * gib) + .saturating_sub(24 * gib) .saturating_sub(headroom); - assert_eq!(available, 4 * gib); - assert!(available >= 1536 * 1024 * 1024); + assert_eq!(available_for_growth, 2 * gib); + assert!(available_for_growth >= 1536 * 1024 * 1024); assert!( pressure From 1ee84d3bf0c16f27046b51053ab946e828d6793b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 07:09:55 +0000 Subject: [PATCH 37/84] fix(admission): make NotApplicable a terminal no-op Closed NotApplicable already finishes the replay record, but the shared pass classifier still backed off on it and the MCP hook boundary still published it as a JSON-RPC failure. Stop that pass without retry or a failure log, and return a successful no-op on the wire. Co-authored-by: Zack Jackson --- CHANGELOG.md | 4 + .../src/profile_host_admission_replay.rs | 41 ++++++++++ .../tracedecay-host-admission/src/replay.rs | 60 +++++++++++++- .../server/project_host_admission_replay.rs | 40 ++++++++++ crates/tracedecay-mcp/src/tool_errors.rs | 78 ++++++++++++++++++- 5 files changed, 221 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc4448d9bf..99bbe0890d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1710,6 +1710,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- *(admission)* `NotApplicable` is a terminal no-op. Replay no longer backs + off on that closed status, and the MCP hook-runtime boundary returns a + successful no-op instead of a JSON-RPC failure. + - *(code-index)* the background worker consults the typed publication-authority park instead of a loop-local bool, so a park it has not yet observed still stops reconcile. Branch publication handles `NotApplicable` as a closed diff --git a/crates/tracedecay-daemon-service/src/profile_host_admission_replay.rs b/crates/tracedecay-daemon-service/src/profile_host_admission_replay.rs index f8d79e2cd4..255e0622ef 100644 --- a/crates/tracedecay-daemon-service/src/profile_host_admission_replay.rs +++ b/crates/tracedecay-daemon-service/src/profile_host_admission_replay.rs @@ -841,6 +841,10 @@ impl ProfileHostAdmissionReplayWorker { // Non-retryable failure: stop until the next explicit kick. break; } + ReplayPassDecision::TerminalNoop => { + consecutive_retryable = 0; + break; + } ReplayPassDecision::Requeue => { consecutive_retryable = 0; } @@ -1391,6 +1395,43 @@ mod tests { registry.shutdown().await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn not_applicable_pending_record_does_not_back_off() { + let temp = tempfile::TempDir::new().unwrap(); + let profile_root = temp.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + let db_path = tracedecay_sessions::runtime::user_sessions_db_path(&profile_root); + let (runtime, _) = + tracedecay_host_admission::HostAdmissionRuntime::open_for_database(&db_path).unwrap(); + let broker = Arc::new(tracedecay_host_admission::HostAdmissionBroker::new(runtime)); + broker.admit("test:pending", b"pending").await.unwrap(); + let registry = ProfileHostAdmissionReplayRegistry::default(); + let pass_override = Arc::new(|| { + Box::pin(async { HostAdmissionOutcome::not_applicable("code_index_not_applicable") }) + as std::pin::Pin + Send>> + }); + + registry + .ensure_with_pass_override(&db_path, &profile_root, &broker, pass_override) + .await; + tokio::time::timeout(Duration::from_secs(1), async { + while registry.pass_count(&db_path).await == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("profile replay must attempt the not-applicable record"); + tokio::time::sleep(Duration::from_millis(150)).await; + let passes = registry.pass_count(&db_path).await; + assert!(passes >= 1); + assert_eq!(registry.backoff_count(&db_path).await, 0); + tokio::time::sleep(Duration::from_millis(150)).await; + + assert_eq!(registry.pass_count(&db_path).await, passes); + assert_eq!(registry.backoff_count(&db_path).await, 0); + registry.shutdown().await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn shutdown_cancels_and_joins_an_in_flight_pass() { let temp = tempfile::TempDir::new().unwrap(); diff --git a/crates/tracedecay-host-admission/src/replay.rs b/crates/tracedecay-host-admission/src/replay.rs index ceffe51442..b9b1588c39 100644 --- a/crates/tracedecay-host-admission/src/replay.rs +++ b/crates/tracedecay-host-admission/src/replay.rs @@ -7,7 +7,7 @@ use std::time::Duration; -use tracedecay_sessions::admission::HostAdmissionOutcome; +use tracedecay_sessions::admission::{HostAdmissionOutcome, HostAdmissionStatus}; const MAX_BACKOFF: Duration = Duration::from_secs(2); const INITIAL_BACKOFF: Duration = Duration::from_millis(25); @@ -28,6 +28,7 @@ pub fn replay_backoff(attempt: u32, shift_cap: u32) -> Duration { } /// How a worker should proceed after one replay pass. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ReplayPassDecision { /// The spool shrank and more work remains. Yield and re-run immediately. ProgressPending, @@ -37,15 +38,33 @@ pub enum ReplayPassDecision { Stop, /// Re-evaluate the work condition without backoff. Requeue, + /// Closed `NotApplicable` left the spool unchanged. Stop until the next + /// kick without backoff and without a failure log. + TerminalNoop, } /// Classify one replay pass from its pending-count delta and outcome. +/// +/// `NotApplicable` already closes the replay record. It must not enter the +/// retryable backoff arm, even when `is_replay_progress` is true and the +/// spool did not shrink: that arm is for work that may succeed later. A shrink +/// with records still pending continues immediately; a drained spool requeues; +/// an unchanged spool stops until the next external kick. pub fn classify_replay_pass( pending_before: usize, pending_after: usize, outcome: &HostAdmissionOutcome, ) -> ReplayPassDecision { let made_progress = pending_after < pending_before; + if outcome.status == HostAdmissionStatus::NotApplicable { + if made_progress && pending_after > 0 { + return ReplayPassDecision::ProgressPending; + } + if pending_after == 0 { + return ReplayPassDecision::Requeue; + } + return ReplayPassDecision::TerminalNoop; + } if made_progress && pending_after > 0 { ReplayPassDecision::ProgressPending } else if !made_progress @@ -58,3 +77,42 @@ pub fn classify_replay_pass( ReplayPassDecision::Requeue } } + +#[cfg(test)] +mod tests { + use super::{ReplayPassDecision, classify_replay_pass}; + use tracedecay_sessions::admission::HostAdmissionOutcome; + + #[test] + fn not_applicable_is_a_terminal_noop_unless_the_spool_actually_moves() { + let closed = HostAdmissionOutcome::not_applicable("code_index_not_applicable"); + let mut flagged_retryable = closed.clone(); + flagged_retryable.retryable = true; + + assert_eq!( + classify_replay_pass(2, 2, &closed), + ReplayPassDecision::TerminalNoop + ); + assert_eq!( + classify_replay_pass(2, 3, &flagged_retryable), + ReplayPassDecision::TerminalNoop + ); + assert_eq!( + classify_replay_pass(2, 1, &closed), + ReplayPassDecision::ProgressPending + ); + assert_eq!( + classify_replay_pass(1, 0, &closed), + ReplayPassDecision::Requeue + ); + + assert_eq!( + classify_replay_pass(2, 2, &HostAdmissionOutcome::accepted_for_replay()), + ReplayPassDecision::Backoff + ); + assert_eq!( + classify_replay_pass(2, 2, &HostAdmissionOutcome::spool_corrupted()), + ReplayPassDecision::Stop + ); + } +} diff --git a/crates/tracedecay-mcp/src/server/project_host_admission_replay.rs b/crates/tracedecay-mcp/src/server/project_host_admission_replay.rs index 6d4f8c5a1d..851e75b05a 100644 --- a/crates/tracedecay-mcp/src/server/project_host_admission_replay.rs +++ b/crates/tracedecay-mcp/src/server/project_host_admission_replay.rs @@ -203,6 +203,10 @@ impl ProjectHostAdmissionReplayWorker { ); break; } + ReplayPassDecision::TerminalNoop => { + consecutive_retryable = 0; + break; + } ReplayPassDecision::Requeue => { consecutive_retryable = 0; if self.dirty.load(Ordering::Acquire) || pending_after > 0 { @@ -275,6 +279,42 @@ mod tests { task.shutdown().await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn not_applicable_pending_record_stops_without_backoff() { + let temp = tempfile::TempDir::new().unwrap(); + let (runtime, _) = tracedecay_host_admission::HostAdmissionRuntime::open( + temp.path(), + tracedecay_host_admission::SpoolBounds::default(), + ) + .unwrap(); + let broker = Arc::new(tracedecay_host_admission::HostAdmissionBroker::new(runtime)); + broker.admit("test:pending", b"pending").await.unwrap(); + let passes = Arc::new(AtomicUsize::new(0)); + let passes_for_run = Arc::clone(&passes); + let pass: PassFn = Arc::new(move || { + let passes = Arc::clone(&passes_for_run); + Box::pin(async move { + passes.fetch_add(1, Ordering::AcqRel); + HostAdmissionOutcome::not_applicable("code_index_not_applicable") + }) + }); + let task = ProjectHostAdmissionReplayTask::start(broker, pass); + + tokio::time::timeout(Duration::from_secs(1), async { + while task.pass_count() == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("project replay must attempt the not-applicable record"); + tokio::time::sleep(Duration::from_millis(100)).await; + + assert_eq!(passes.load(Ordering::Acquire), 1); + assert_eq!(task.pass_count(), 1); + assert_eq!(task.backoff_count(), 0); + task.shutdown().await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn dropping_task_aborts_an_in_flight_pass_without_an_arc_cycle() { let temp = tempfile::TempDir::new().unwrap(); diff --git a/crates/tracedecay-mcp/src/tool_errors.rs b/crates/tracedecay-mcp/src/tool_errors.rs index 758792e860..e0ab8cd06c 100644 --- a/crates/tracedecay-mcp/src/tool_errors.rs +++ b/crates/tracedecay-mcp/src/tool_errors.rs @@ -44,6 +44,28 @@ fn value_has_semantic_error(value: &Value) -> bool { }) } +/// Successful MCP tool result for a closed `NotApplicable` admission. +/// +/// The status already means the demand does not apply. Publishing it as a +/// JSON-RPC error makes hosts retry a terminal no-op. +fn hook_runtime_not_applicable_noop(error: &TraceDecayError) -> Option { + let status = error.hook_runtime_status()?; + if HostAdmissionStatus::from_wire(status) != Some(HostAdmissionStatus::NotApplicable) { + return None; + } + let (reason_code, _, _) = error.hook_runtime_context()?; + let body = json!({ + "tool": "tracedecay_hook_runtime", + "status": status, + "reason_code": reason_code, + "retryable": false, + }); + Some(json!({ + "content": [{ "type": "text", "text": body.to_string() }], + "isError": false, + })) +} + /// Projects a hook-runtime error onto the structured JSON-RPC data object. /// /// The status is whatever the admission authority reported, carried through @@ -169,6 +191,11 @@ pub fn tool_error_response(id: Value, tool_name: &str, error: &TraceDecayError) Some(data), ); } + if tool_name == "tracedecay_hook_runtime" + && let Some(result) = hook_runtime_not_applicable_noop(error) + { + return JsonRpcResponse::success(id, result); + } if tool_name == "tracedecay_hook_runtime" && let Some(data) = structured_hook_error_data(error) { @@ -342,7 +369,7 @@ pub fn serialize_response_line(resp: &JsonRpcResponse) -> String { #[cfg(test)] mod tests { - use serde_json::json; + use serde_json::{Value, json}; use tracedecay_domain::errors::TraceDecayError; use super::tool_error_response; @@ -367,4 +394,53 @@ mod tests { "application_surface_invalid_request" ); } + + #[test] + fn not_applicable_hook_runtime_is_a_successful_noop_and_other_statuses_stay_errors() { + use tracedecay_sessions::admission::HostAdmissionStatus; + + let closed = TraceDecayError::hook_runtime_with_status( + "code_index_not_applicable", + true, + "projectless Hermes receipt host admission failed", + HostAdmissionStatus::NotApplicable.as_wire(), + ); + let wire = serde_json::to_value(tool_error_response( + json!(4), + "tracedecay_hook_runtime", + &closed, + )) + .expect("JSON-RPC wire response"); + + assert_eq!(wire["jsonrpc"], "2.0"); + assert_eq!(wire["id"], 4); + assert!(wire.get("error").is_none()); + assert_eq!(wire["result"]["isError"], false); + let text = wire["result"]["content"][0]["text"] + .as_str() + .expect("tool text"); + let body: Value = serde_json::from_str(text).expect("noop body"); + assert_eq!(body["tool"], "tracedecay_hook_runtime"); + assert_eq!(body["status"], "not_applicable"); + assert_eq!(body["reason_code"], "code_index_not_applicable"); + assert_eq!(body["retryable"], false); + assert!(body.get("error").is_none()); + assert!(!text.contains("failed")); + + let unavailable = TraceDecayError::hook_runtime_with_status( + "authority_unavailable", + true, + "host admission authority is unavailable", + HostAdmissionStatus::Unavailable.as_wire(), + ); + let failure = serde_json::to_value(tool_error_response( + json!(5), + "tracedecay_hook_runtime", + &unavailable, + )) + .expect("JSON-RPC wire response"); + assert!(failure.get("result").is_none()); + assert_eq!(failure["error"]["code"], -32603); + assert_eq!(failure["error"]["data"]["status"], "unavailable"); + } } From 23c96f2dcd99aaf35741074d51e1b2c2ee93ce21 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 07:16:11 +0000 Subject: [PATCH 38/84] ci(partitions): register the hotpath guard crate's targets `scripts/linux-test-partitions.py check`, the `scope-gate` job's proof that no test target falls out of the Linux lane, failed on this branch: the new `tracedecay-hotpath-guard` crate's lib and its `functions_limit_live` test were in no partition. The lib joins `core-contracts` beside the other leaf crates. The live MCP proof needs `hotpath` and `hotpath-mcp`, which no test lane enables, so it is listed under `not_run` the way `tracedecay-global-db::schema_convergence_hotpath` already is; the `cargo check --workspace --all-targets --features hotpath,hotpath-mcp` gate still compiles it. Co-Authored-By: Claude Fable 5.1 --- .github/linux-test-partitions.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/linux-test-partitions.json b/.github/linux-test-partitions.json index 9febd5785a..313c8c6150 100644 --- a/.github/linux-test-partitions.json +++ b/.github/linux-test-partitions.json @@ -193,6 +193,7 @@ "tracedecay-framing", "tracedecay-hooks", "tracedecay-host-integration", + "tracedecay-hotpath-guard", "tracedecay-mcp-catalog", "tracedecay-policy", "tracedecay-privacy", @@ -236,6 +237,7 @@ } ], "not_run": { - "tracedecay-global-db::schema_convergence_hotpath": "requires hotpath-alloc, which no test lane enables; the hotpath workflows own it" + "tracedecay-global-db::schema_convergence_hotpath": "requires hotpath-alloc, which no test lane enables; the hotpath workflows own it", + "tracedecay-hotpath-guard::functions_limit_live": "requires hotpath and hotpath-mcp to start a live MCP server, which no test lane enables; the hotpath graph check compiles it" } } From d36517e46f10f5d40748214ea1cbe003b05b1364 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 07:16:12 +0000 Subject: [PATCH 39/84] test(hotpath-guard): cover the functions limit env fallback `functions_display_limit` picks `HOTPATH_FUNCTIONS_LIMIT`, else `HOTPATH_LIMIT`, and treats an unparsable value as unset. The only proof of that chain was `tests/functions_limit_live.rs`, which needs `hotpath` and `hotpath-mcp` and therefore never runs in a test lane, so the crate's new `core-contracts` entry ran nothing. One lib test owns both variables, because they are process-global, and covers precedence, the `0` unlimited case, and junk in either slot. Co-Authored-By: Claude Fable 5.1 --- crates/tracedecay-hotpath-guard/src/lib.rs | 47 ++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/crates/tracedecay-hotpath-guard/src/lib.rs b/crates/tracedecay-hotpath-guard/src/lib.rs index 1c09f45fb8..89121cb917 100644 --- a/crates/tracedecay-hotpath-guard/src/lib.rs +++ b/crates/tracedecay-hotpath-guard/src/lib.rs @@ -27,3 +27,50 @@ fn functions_display_limit() -> Option { fn parse_usize_env(name: &str) -> Option { std::env::var(name).ok().and_then(|raw| raw.parse().ok()) } + +#[cfg(test)] +mod tests { + use super::functions_display_limit; + + /// One test owns both variables: they are process-global, and the live MCP + /// proof (`tests/functions_limit_live.rs`) only runs under `hotpath`. + #[test] + fn functions_limit_precedes_the_global_limit_and_ignores_junk() { + let set = |name: &str, value: Option<&str>| unsafe { + match value { + Some(value) => std::env::set_var(name, value), + None => std::env::remove_var(name), + } + }; + + set("HOTPATH_FUNCTIONS_LIMIT", None); + set("HOTPATH_LIMIT", None); + assert_eq!(functions_display_limit(), None, "unset leaves the builder"); + + set("HOTPATH_LIMIT", Some("7")); + assert_eq!(functions_display_limit(), Some(7), "HOTPATH_LIMIT is used"); + + set("HOTPATH_FUNCTIONS_LIMIT", Some("2")); + assert_eq!( + functions_display_limit(), + Some(2), + "HOTPATH_FUNCTIONS_LIMIT wins" + ); + + set("HOTPATH_FUNCTIONS_LIMIT", Some("0")); + assert_eq!(functions_display_limit(), Some(0), "0 stays unlimited"); + + set("HOTPATH_FUNCTIONS_LIMIT", Some("not-a-number")); + assert_eq!( + functions_display_limit(), + Some(7), + "junk falls back to HOTPATH_LIMIT" + ); + + set("HOTPATH_LIMIT", Some("-1")); + assert_eq!(functions_display_limit(), None, "junk in both leaves it"); + + set("HOTPATH_FUNCTIONS_LIMIT", None); + set("HOTPATH_LIMIT", None); + } +} From bf444b5119630a711bc8ff32bcbb1e2d55443269 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 07:17:01 +0000 Subject: [PATCH 40/84] test(daemon): await the released descendant before probing `group_stop_releases_an_inherited_listen_socket` failed 2 of 40 local runs of the freshly built `daemon_suite` binary with `connect returned Ok(UnixStream ...)`. The proof spawns a holder that binds, listens and forks, then asserts the path is refused the instant `kill_and_wait` returns. That assertion is stronger than the kernel guarantees. The descendant holding the inherited listen descriptor is reparented, not a child, so the harness cannot `wait` on it: the group `SIGKILL` is delivered asynchronously and its descriptors close when the kernel finishes the teardown, not when the leader's `wait` returns. The harness does not rely on that instant either - `release_socket_on_stop` unlinks the published path, which is what makes `spawn_tracedecay_daemon_process` deterministic. Poll for the refusal on a bounded deadline. A descendant that was never signaled keeps accepting past it, so the regression this proof exists to catch still fails the test. Co-Authored-By: Claude Fable 5.1 --- .../daemon_suite/socket_lifecycle_test.rs | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay/tests/daemon_suite/socket_lifecycle_test.rs b/crates/tracedecay/tests/daemon_suite/socket_lifecycle_test.rs index aa1eed3648..bd2f0f1dbc 100644 --- a/crates/tracedecay/tests/daemon_suite/socket_lifecycle_test.rs +++ b/crates/tracedecay/tests/daemon_suite/socket_lifecycle_test.rs @@ -9,7 +9,16 @@ use std::os::unix::process::CommandExt; use std::process::{Command, Stdio}; use std::time::{Duration, Instant}; -use crate::common::TestChildProcess; +use crate::common::{TestChildProcess, poll_until}; + +/// How long the released descendant has to finish dying. +/// +/// The group signal is delivered to a process the harness cannot `wait` on - +/// the descendant is reparented, not a child - so its descriptors close when +/// the kernel finishes tearing it down, not when the leader's `wait` returns. +/// A descendant that was never signaled keeps accepting past this deadline, +/// which is the regression this proof exists to catch. +const DESCENDANT_RELEASE_TIMEOUT: Duration = Duration::from_secs(10); const HOLDER: &str = r#" import os, socket, time @@ -58,10 +67,16 @@ fn group_stop_releases_an_inherited_listen_socket() { socket.exists(), "this proof must not delete the socket path" ); - let refused = UnixStream::connect(&socket); - assert!( - refused.is_err(), - "process-group stop must release the inherited listen socket, connect returned {refused:?}" + poll_until( + Instant::now() + DESCENDANT_RELEASE_TIMEOUT, + Duration::from_millis(20), + || UnixStream::connect(&socket).is_err().then_some(()), + || { + format!( + "process-group stop must release the inherited listen socket at {}", + socket.display() + ) + }, ); } From 4e200b277371a866cc4cea60e72eaaa0d878ab86 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 07:19:30 +0000 Subject: [PATCH 41/84] fix(ci): keep evaluator acceptance tests on the Windows lane Gating `packaged_search_evaluator` and `search_eval_cli_test` behind `tracedecay/search-eval` removed them from every build that enables only `tracedecay/test-helpers`. The Windows lane and the `test-ci` alias are such builds, so their five shards stopped checking CLI/library receipt parity, conceptual misses, invalid-fixture failure behavior and packaged execution while still provisioning the evaluator binaries. Both selections build `--workspace`, which already resolves `tracedecay-search-eval` and `tracedecay-query/search-eval`, so enabling the root feature there restores the four tests without compiling anything new. The Linux partitions split the package selection, which is why the feature stays per-partition there. Co-Authored-By: Claude Fable 5.1 --- .cargo/config.toml | 14 +++++++++----- .github/workflows/ci.yml | 13 +++++++++---- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index ef2197b4ef..ce65ee3f3a 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -67,11 +67,15 @@ rustflags = ["-C", "split-debuginfo=unpacked"] # # `test-ci` is the hosted acceptance selection: nextest's `ci` policy # (.config/nextest.toml: fail-fast off, one retry that still fails flaky -# results, 8 threads, slow-timeout termination) and only the root fixture -# feature. `.github/workflows/ci.yml` runs this exact alias, and its support -# binaries/archives are built with `--profile perf` so they share artifacts -# with it. Extra flags append: `cargo test-ci --locked -E 'binary(=x)'`. -test-ci = "nextest run --workspace --profile ci --features tracedecay/test-helpers --cargo-profile perf" +# results, 8 threads, slow-timeout termination), the root fixture feature and +# `search-eval`, which is what compiles the evaluator acceptance suites (the +# root crate's dependency on the evaluator is optional so the Linux transport +# partition does not build the eval-only lexical projection; a `--workspace` +# run resolves the evaluator anyway). `.github/workflows/ci.yml` runs this +# exact alias, and its support binaries/archives are built with `--profile +# perf` so they share artifacts with it. Extra flags append: `cargo test-ci +# --locked -E 'binary(=x)'`. +test-ci = "nextest run --workspace --profile ci --features tracedecay/test-helpers,tracedecay/search-eval --cargo-profile perf" # Deliberately broader than CI: every optional feature (test-transport # acceptance suites, hotpath, …) under the same cargo profile, with the # default nextest policy (no retries). diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 004961420a..b58e92e998 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -741,7 +741,12 @@ jobs: shared-key: ci-test-full-windows-msvc-lld cache-on-failure: true - # Acceptance tests execute the ordinary evaluator binaries. Match the + # Acceptance tests execute the ordinary evaluator binaries, and + # `tracedecay/search-eval` is what compiles the suites that check the + # CLI receipt against the library. This lane builds `--workspace`, so + # the evaluator and `tracedecay-query/search-eval` already resolve here + # and the feature adds no compilation; the Linux partitions split the + # package selection, which is why it is per-partition there. Match the # archive's features and perf profile to reuse dependency artifacts; # nextest's archive.include carries the executables to every shard. # `--tests` keeps the binaries in the dev-dependency graph the archive @@ -750,14 +755,14 @@ jobs: # lane). - name: Build workspace binaries and tests for the Windows test lane shell: pwsh - run: cargo build --workspace --bins --tests --locked --profile perf --features tracedecay/test-helpers + run: cargo build --workspace --bins --tests --locked --profile perf --features tracedecay/test-helpers,tracedecay/search-eval # `--workspace`, not `-p tracedecay-cli`: the package selection decides # feature unification, and the narrower one recompiles the code-index # and extraction crates in a second configuration (see the Linux lane). - name: Build Windows host-CLI test fixture shell: pwsh - run: cargo build --workspace --example tracedecay-host-cli-fixture --locked --profile perf --features tracedecay/test-helpers + run: cargo build --workspace --example tracedecay-host-cli-fixture --locked --profile perf --features tracedecay/test-helpers,tracedecay/search-eval # The controlled-workload Hotpath parity helpers are provisioned and # verified by the `hotpath-parity` job. Building them here would put a @@ -768,7 +773,7 @@ jobs: # archive alias, so the arguments are spelled out here once. - name: Build nextest archive shell: pwsh - run: cargo nextest archive --workspace --profile ci --locked --features tracedecay/test-helpers --cargo-profile perf --timings --archive-file "$env:RUNNER_TEMP/nextest-archive.tar.zst" + run: cargo nextest archive --workspace --profile ci --locked --features tracedecay/test-helpers,tracedecay/search-eval --cargo-profile perf --timings --archive-file "$env:RUNNER_TEMP/nextest-archive.tar.zst" - name: Upload Windows archive build timings if: always() From b8a12f86c05588a7e003cfb6e928fbcc8e25a430 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 07:22:29 +0000 Subject: [PATCH 42/84] test(lcm): expect the payload placeholder describe now returns lcm_describe_supports_summary_node_and_external_payload_targets still required an empty external-payload preview, so it failed once describe started returning the stored placeholder. Assert the placeholder names the payload ref instead; the body-leak assertions below are unchanged. Co-Authored-By: Claude Fable 5.1 --- .../tests/mcp_suite/mcp_handler_test/lcm_test.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_test.rs index 5e02ef225c..3769ae4fdb 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_test.rs @@ -754,9 +754,12 @@ async fn lcm_describe_supports_summary_node_and_external_payload_targets() { payload_payload["description"]["external_payload"]["payload_ref"], payload_ref ); - assert_eq!( - payload_payload["description"]["external_payload"]["content_preview"], - "" + let payload_preview = payload_payload["description"]["external_payload"]["content_preview"] + .as_str() + .unwrap_or_else(|| panic!("payload describe preview missing: {payload_payload}")); + assert!( + payload_preview.contains(payload_ref.as_str()), + "payload describe must return the stored placeholder: {payload_preview:?}" ); assert_eq!(payload_payload["grain"], "occurrence"); assert_eq!(payload_payload["state"], "available"); From c9a371a0f6acf4d76c353e6c97690bd080b7d48a Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 07:23:22 +0000 Subject: [PATCH 43/84] style(daemon): rewrap the store import after the error drop The cursor-frontier change removed ObservationStoreError from the store_runtime_tests import list but left the old wrapping, so `cargo fmt --all -- --check` failed on that block. Co-Authored-By: Claude Fable 5.1 --- crates/tracedecay/src/daemon/store_runtime_tests.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay/src/daemon/store_runtime_tests.rs b/crates/tracedecay/src/daemon/store_runtime_tests.rs index b60d6f0f74..b0bae69db0 100644 --- a/crates/tracedecay/src/daemon/store_runtime_tests.rs +++ b/crates/tracedecay/src/daemon/store_runtime_tests.rs @@ -30,9 +30,9 @@ use tracedecay_session_memory::memory::{ }; use tracedecay_store::{ CursorAdvanceOutcome, FactReadControl, FactWriteControl, ObservationCoverageReason, - ObservationCursorAdvance, ObservationStore, ProjectId, - ProjectMemoryFactHistoryQueryV1, ProjectMemoryFactIdV1, ProjectMemoryFactProjectionV1, - RetainedGraphStoreLeaseV1, StoreShardIdV1, + ObservationCursorAdvance, ObservationStore, ProjectId, ProjectMemoryFactHistoryQueryV1, + ProjectMemoryFactIdV1, ProjectMemoryFactProjectionV1, RetainedGraphStoreLeaseV1, + StoreShardIdV1, }; use tracedecay_store_runtime::{ DaemonSessionRuntimeRegistryV1, RegisteredSchemaConvergenceStatus, process_runtime_generation, From ed6d8f9f1a24d780354045b653703ea835926a6c Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 07:23:31 +0000 Subject: [PATCH 44/84] style(code-index): use map_or for the pointer memo size `cargo clippy --workspace --all-targets --locked -- -D warnings` failed on `map_unwrap_or` in the pointer memo installed by `remember_publication_pointer`, which is CI's exact lens. Co-Authored-By: Claude Fable 5.1 --- .../src/code_index_scheduler/publication_store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs index d52e0c3962..9be2ec5f7f 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs @@ -1261,7 +1261,7 @@ impl DaemonCodeIndexPublicationStoreV1 { mtime: metadata .as_ref() .and_then(|metadata| metadata.modified().ok()), - size: metadata.map(|metadata| metadata.len()).unwrap_or(0), + size: metadata.map_or(0, |metadata| metadata.len()), digest: Self::state_digest(bytes), pointer: pointer.clone(), }); From 7725c98e019d158a1aba8288d56521a96f5fe447 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 07:29:31 +0000 Subject: [PATCH 45/84] fix(search-eval): keep the production fallback contract check Replacing the fallback digest with a ranking receipt also removed the per-query `QueryFallbackSubpayload::new` call, and with it the only assertion that each composed query has canonical `final_ordinal` ordering, valid candidates, and query-fallback-only contributions. search-eval composes through `CompositionKernel` directly, so `query_authority`'s own construction never covers this path. Reconstruct the subpayload for its contract and discard the digest; the ranking receipt stays the pin. Co-Authored-By: Claude Fable 5.1 --- .../src/candidate_output.rs | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-search-eval/src/candidate_output.rs b/crates/tracedecay-search-eval/src/candidate_output.rs index 43d6772e12..3ce4a90842 100644 --- a/crates/tracedecay-search-eval/src/candidate_output.rs +++ b/crates/tracedecay-search-eval/src/candidate_output.rs @@ -44,11 +44,12 @@ use tracedecay_domain::{ EphemeralSanitizedQueryViewV1, ExactAdmissionRuleRevision, ExactClass, FileOccurrenceId, LanguageId, ManifestDigest, PolicyRevisionId, PrincipalId, PrivacyDomainId, ProjectId, ProjectionBatchRequestV1, ProjectionKeyV1, ProjectionKindV1, ProjectionOperationV1, - ProjectionOutcomeV1, PublicRetrieverStatus, QueryNormalizationRevision, RelationEdgeKindV1, - RepositoryDirtyStateV1, RepositoryId, RetrievalFailure, RetrievalRequest, RetrievalScope, - RetrievalSnapshot, RetrieverKind, RetrieverOutcome, SanitizationReceiptId, SanitizedCodeFileV1, - SanitizedCodeSnapshotV1, SanitizerRevision, SingleRootScopeV1, SnapshotFileDispositionV1, - SymbolOccurrenceId, TemporalModeV1, UtcMicros, VectorWatermark, + ProjectionOutcomeV1, PublicRetrieverStatus, QueryFallbackSubpayload, + QueryNormalizationRevision, RelationEdgeKindV1, RepositoryDirtyStateV1, RepositoryId, + RetrievalFailure, RetrievalRequest, RetrievalScope, RetrievalSnapshot, RetrieverKind, + RetrieverOutcome, SanitizationReceiptId, SanitizedCodeFileV1, SanitizedCodeSnapshotV1, + SanitizerRevision, SingleRootScopeV1, SnapshotFileDispositionV1, SymbolOccurrenceId, + TemporalModeV1, UtcMicros, VectorWatermark, }; use tracedecay_query::native_git::NativeHistoricalBlobReaderV1; use tracedecay_query::retrieval::exact::{ @@ -599,6 +600,18 @@ fn generate_partition_output( let composed = compose_production_query(published, profile, query)?; let ranked = map_ranked_candidates(published, &composed)?; let coverage = query_lane_coverage(&composed); + // The subpayload's digest is unfit as a ranking pin; its contract is + // not. Constructing it still proves canonical `final_ordinal` order, + // per-candidate validity, and query-fallback-only contributions for + // every composed query. + QueryFallbackSubpayload::new( + composed.profile_id.clone(), + composed.ranked_candidates.clone(), + coverage.clone(), + composed.freshness.clone(), + None, + ) + .map_err(|error| CandidateOutputError::Contract(error.to_string()))?; let receipt = ranking_receipt_digest( composed.profile_id.as_str(), query.query_id.as_str(), From 3daba38304cee844b9ffba128a7a0b58f72bf368 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 07:35:20 +0000 Subject: [PATCH 46/84] fix(extraction): keep module scope in self receiver types `enclosing_receiver_type` returned the bare impl or trait owner, so a `self.len()` inside `mod inner { impl Rows { .. } }` emitted `Rows::len` while the definition is indexed under its whole file-relative name, `inner::Rows::len`. `resolve_file_references` exact-matches that key, so with the bare method-name duplicate gone the call bound nothing at all and was retained as a phantom cross-file reference instead. The receiver type now carries the frames between the file root and the impl, which is exactly how the method's own qualified name is built. The stored owner of a trait impl is ``, and `Type` can itself be a projection (`::Item`), so splitting at the first ` as ` yielded ` --- .../src/rust_extractor.rs | 64 ++++++++++++++----- .../tests/main/rust.rs | 60 +++++++++++++++++ 2 files changed, 109 insertions(+), 15 deletions(-) diff --git a/crates/tracedecay-code-extraction/src/rust_extractor.rs b/crates/tracedecay-code-extraction/src/rust_extractor.rs index 17261a1a99..c00f740eda 100644 --- a/crates/tracedecay-code-extraction/src/rust_extractor.rs +++ b/crates/tracedecay-code-extraction/src/rust_extractor.rs @@ -1663,25 +1663,22 @@ impl RustExtractor { Some(format!("{type_path}::{}", state.node_text(field))) } - /// The type `self` names in the enclosing impl or trait. + /// The type `self` names in the enclosing impl or trait, carrying the + /// enclosing module path. /// /// Trait impls store `` so the method keeps a UFCS name. /// `self` still names `Type`, the path a call site writes and the alias - /// same-file resolution binds. + /// same-file resolution binds. Same-file resolution keys a definition by + /// its file-relative qualified name, so an impl inside `mod inner` has to + /// name `inner::Type::method` or the call binds nothing. fn enclosing_receiver_type(state: &ExtractionState<'_>) -> Option { - let (name, id) = state + let owner = state .node_stack .iter() - .rev() - .find(|(_, id)| id.starts_with("impl:") || id.starts_with("trait:"))?; + .rposition(|(_, id)| id.starts_with("impl:") || id.starts_with("trait:"))?; + let (name, id) = &state.node_stack[owner]; let type_name = if id.starts_with("impl:") { - match name - .strip_prefix('<') - .and_then(|inner| inner.split_once(" as ")) - { - Some((type_name, _)) => type_name.trim(), - None => name.as_str(), - } + Self::impl_owner_type_name(name) } else { name.as_str() }; @@ -1690,10 +1687,47 @@ impl RustExtractor { || type_name == "" || type_name == "" { - None - } else { - Some(type_name.to_owned()) + return None; + } + // Frame 0 is the file root, which the qualified name drops. + let mut path = state + .node_stack + .get(1..owner) + .unwrap_or_default() + .iter() + .map(|(segment, _)| segment.as_str()) + .collect::>(); + path.push(type_name); + Some(path.join("::")) + } + + /// The self type inside a stored impl owner name. + /// + /// A trait impl stores ``, and `Type` can itself be a + /// projection (`::Item`), so the delimiter is the ` as ` at + /// depth zero inside the wrapper, not the first one in the string. + fn impl_owner_type_name(owner: &str) -> &str { + let Some(inner) = owner.strip_prefix('<') else { + return owner; + }; + let mut depth = 0_i32; + for (index, character) in inner.char_indices() { + match character { + '<' => depth += 1, + '>' => { + if depth == 0 { + break; + } + depth -= 1; + } + _ => { + if depth == 0 && inner[index..].starts_with(" as ") { + return inner[..index].trim(); + } + } + } } + owner } /// Records every binding the function introduces with the type it states, diff --git a/crates/tracedecay-code-extraction/tests/main/rust.rs b/crates/tracedecay-code-extraction/tests/main/rust.rs index 7e74529f99..664d131ec8 100644 --- a/crates/tracedecay-code-extraction/tests/main/rust.rs +++ b/crates/tracedecay-code-extraction/tests/main/rust.rs @@ -1237,6 +1237,66 @@ fn make() -> Vec { Vec::new() } ); } +#[test] +fn self_receiver_names_carry_module_scope_and_the_outer_as_delimiter() { + let source = r#" +mod inner { + pub struct Rows; + impl Rows { + fn len(&self) -> usize { 0 } + fn measure(&self) -> usize { self.len() } + } + trait Wide { fn wide(&self) -> usize; } + impl Wide for Rows { + fn wide(&self) -> usize { self.len() } + } +} +struct Foo; +trait Assoc { type Item; } +trait Local { fn span(&self) -> usize; } +impl Local for ::Item { + fn span(&self) -> usize { self.len() } +} +"#; + let result = RustExtractor.extract("src/lib.rs", source); + assert!(result.errors.is_empty(), "{:?}", result.errors); + + let from = |qualified: &str| { + let function = result + .nodes + .iter() + .find(|node| { + matches!(node.kind, NodeKind::Function | NodeKind::Method) + && node.qualified_name == qualified + }) + .unwrap_or_else(|| panic!("{qualified} is extracted")); + result + .unresolved_refs + .iter() + .filter(|reference| { + reference.reference_kind == EdgeKind::Calls && reference.from_node_id == function.id + }) + .map(|reference| reference.reference_name.as_str()) + .collect::>() + }; + + let measure = from("src/lib.rs::inner::Rows::measure"); + assert!( + measure.contains(&"inner::Rows::len"), + "self inside `mod inner` names the module-scoped type: {measure:?}" + ); + let wide = from("src/lib.rs::inner::::wide"); + assert!( + wide.contains(&"inner::Rows::len"), + "a trait impl in a module keeps the module path: {wide:?}" + ); + let span = from("src/lib.rs::<::Item as Local>::span"); + assert!( + span.contains(&"::Item::len"), + "a projected self type splits at the outer `as`: {span:?}" + ); +} + #[test] fn wildcard_imports_retain_unresolved_dependencies_alongside_named_bindings() { let result = RustExtractor.extract( From f06027655d70d82a0b9e88c35dcdba3d1f0ddcfe Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 07:35:32 +0000 Subject: [PATCH 47/84] fix(code-index): alias module-scoped trait impl methods `rust_type_path_alias_for_trait_impl_method` required the file-relative name to start with `<`, so a trait impl inside an inline module (`inner::::wide`) got no `inner::Rows::wide` alias. That was invisible while every dotted call also emitted its bare method name; now that a call names its receiver type, `self.wide()` in that module bound nothing. The parser now peels an enclosing module path before the UFCS head and restores it on the alias, so the guard that lets an inherent method keep the path and `rust_qualified_name_is_trait_impl_method` see the same shape both keep working for module-scoped impls. Co-Authored-By: Claude Fable 5.1 --- crates/tracedecay-code-index/src/chunks.rs | 78 +++++++++++++++++++++- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-code-index/src/chunks.rs b/crates/tracedecay-code-index/src/chunks.rs index 85f27ed90c..08e9be3684 100644 --- a/crates/tracedecay-code-index/src/chunks.rs +++ b/crates/tracedecay-code-index/src/chunks.rs @@ -2102,8 +2102,14 @@ pub(crate) fn cross_file_reference_name_is_blocklisted(reference_name: &str) -> /// `Builder::default`). Keeps the intentional `` definition /// name while restoring same-file / seal recall for those calls. `None` when /// `path` is not a well-formed UFCS trait-impl method. +/// +/// A trait impl inside an inline module carries that module path +/// (`inner::::wide`); the alias keeps it, because same-file +/// resolution keys definitions by their whole file-relative name. pub(crate) fn rust_type_path_alias_for_trait_impl_method(path: &str) -> Option { - if !path.starts_with('<') { + let open = path.find('<')?; + let (module_prefix, path) = path.split_at(open); + if !(module_prefix.is_empty() || module_prefix.ends_with("::")) { return None; } let mut depth = 0_i32; @@ -2139,7 +2145,7 @@ pub(crate) fn rust_type_path_alias_for_trait_impl_method(path: &str) -> Option usize; }\n", + " impl Wide for Rows {\n", + " fn wide(&self) -> usize { 1 }\n", + " }\n", + " impl Rows {\n", + " fn len(&self) -> usize { 0 }\n", + " fn measure(&self) -> usize { self.len() + self.wide() }\n", + " }\n", + "}\n", + ); + let file = validated_file("src/lib.rs", source.as_bytes()); + let batch = batch_for(&file, ParseOutcomeV1::Complete); + let artifacts = chunker() + .index_file(&file, &batch, &rust_descriptor(), &NeverCancelled) + .expect("indexing succeeds"); + let qualified = |occurrence: &SymbolOccurrenceId| { + artifacts + .symbols + .iter() + .find(|symbol| &symbol.occurrence == occurrence) + .map(|symbol| symbol.qualified_name.as_str()) + .unwrap_or("") + }; + let mut calls = artifacts + .edges + .iter() + .filter(|edge| edge.kind == RelationEdgeKindV1::Calls) + .map(|edge| { + ( + qualified(&edge.from_occurrence).to_owned(), + qualified(&edge.to_occurrence).to_owned(), + ) + }) + .collect::>(); + calls.sort(); + + assert_eq!( + calls, + vec![ + ( + "src/lib.rs::inner::Rows::measure".to_owned(), + "src/lib.rs::inner::::wide".to_owned(), + ), + ( + "src/lib.rs::inner::Rows::measure".to_owned(), + "src/lib.rs::inner::Rows::len".to_owned(), + ), + ], + "`self` must name the module-scoped receiver type so same-file \ + resolution still binds: {calls:?}" + ); + let retained_calls = artifacts + .unresolved_references + .iter() + .filter(|reference| reference.kind == RelationEdgeKindV1::Calls) + .collect::>(); + assert!( + retained_calls.is_empty(), + "a same-file self call must not be retained as cross-file: \ + {retained_calls:?}" + ); + } + #[test] fn rust_type_path_alias_parses_ufcs_trait_impl_methods() { assert_eq!( From 1ebadc81f9c6ba7c1263ec863bed027d5a057c70 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 07:35:44 +0000 Subject: [PATCH 48/84] chore(code-index): move Rust extraction to extractor.rust.v11 The extractor no longer emits the bare method name of a dotted call and now types `self` from the enclosing impl or trait, so the same bytes produce different reference rows. `generation_language_revisions_match` compares a sealed generation's extractor revisions against the registry, so without a bump every already-sealed generation keeps the false same-file callers this change exists to remove until some unrelated edit forces re-extraction. Re-pin the three fixtures that carry the revision string: - `canonical_rows_digest_matches_pinned_identity`: the revision is part of the batch identity, so the pinned rows digest moves to sha256:e92b7ad8f9. - `partitioned_codec_has_stable_bytes_and_round_trips`: as in acc2773e90 ("re-pin partitioned codec onto extractor.rust.v10") the revision sits in every sealed file segment. v10 and v11 are the same length, so all four segment sizes are unchanged (11_071, 5_171, 6_279, 6_837) and only the digests move: an identity change, not container drift. - The two worker tests and the reconcile test that assert the current revision after a forced re-extraction. Co-Authored-By: Claude Fable 5.1 --- .../src/code_index_scheduler/tests/reconcile.rs | 2 +- crates/tracedecay-code-index/src/extract.rs | 9 +++++---- crates/tracedecay-code-index/src/languages.rs | 11 +++++++---- .../src/production/worker_tests.rs | 4 ++-- .../code_index_suite/production_orchestration.rs | 10 +++++----- 5 files changed, 20 insertions(+), 16 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs index 0ac4e3a71f..44e4d9f5b1 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs @@ -851,7 +851,7 @@ fn retained_stale_rust_extractor_generation_is_refused_and_rebuilt() { .iter() .find(|(language, _)| language.as_str() == "rust") .map(|(_, revision)| revision.as_str()), - Some("extractor.rust.v10") + Some("extractor.rust.v11") ); } diff --git a/crates/tracedecay-code-index/src/extract.rs b/crates/tracedecay-code-index/src/extract.rs index 85eb041ceb..cd4f39eb1a 100644 --- a/crates/tracedecay-code-index/src/extract.rs +++ b/crates/tracedecay-code-index/src/extract.rs @@ -776,12 +776,13 @@ mod tests { // extractor.rust.v8 records restricted `pub` re-export scope as a // typed value and no longer fabricates receiver types for method // initializers; v9 adds the clone-body token bound and v10 the byte - // bound. The revision is part of the batch identity, so the pinned - // digest moves with it. - assert_eq!(descriptor.extractor_revision.as_str(), "extractor.rust.v10"); + // bound; v11 drops the bare method name of a dotted call and types + // `self` from the enclosing impl or trait. The revision is part of the + // batch identity, so the pinned digest moves with it. + assert_eq!(descriptor.extractor_revision.as_str(), "extractor.rust.v11"); assert_eq!( extraction.batch().rows_digest.as_str(), - "sha256:2e1ebb8fbd7b438059eda5da2db9c8db9707f219484a28e82066caf527b038c6" + "sha256:e92b7ad8f93e3576c70adafb0690d064bd996c207a4ecd7b855c96e3ad3959ad" ); } diff --git a/crates/tracedecay-code-index/src/languages.rs b/crates/tracedecay-code-index/src/languages.rs index 9f3a01714c..5d7ea779ed 100644 --- a/crates/tracedecay-code-index/src/languages.rs +++ b/crates/tracedecay-code-index/src/languages.rs @@ -210,10 +210,13 @@ impl StaticLanguageRegistry { // the owning type. Every language moved one revision when clone-body // eligibility gained its token bound and again when it gained the // pre-tokenization byte bound: one multi-megabyte literal is only - // a few tokens but still cannot fit a text-artifact page. Only - // re-extraction removes the poisoned record. + // a few tokens but still cannot fit a text-artifact page. Rust v11 + // stopped emitting the bare method name of a dotted call, so an + // unrelated same-file callable sharing that name is no longer a + // caller, and types `self` through the enclosing impl or trait. + // Only re-extraction removes the poisoned record. let extractor_revision = if language == "rust" { - 10 + 11 } else if matches!(language.as_str(), "typescript" | "protobuf" | "sql") { 6 } else { @@ -418,7 +421,7 @@ mod tests { assert!(rust.stable_member_spans); assert!(rust.capabilities.extraction); assert_eq!(rust.root_markers, vec!["Cargo.toml".to_owned()]); - assert_eq!(rust.extractor_revision.as_str(), "extractor.rust.v10"); + assert_eq!(rust.extractor_revision.as_str(), "extractor.rust.v11"); assert_eq!( registry diff --git a/crates/tracedecay-code-index/src/production/worker_tests.rs b/crates/tracedecay-code-index/src/production/worker_tests.rs index cda982bd16..7531f94588 100644 --- a/crates/tracedecay-code-index/src/production/worker_tests.rs +++ b/crates/tracedecay-code-index/src/production/worker_tests.rs @@ -357,7 +357,7 @@ fn extractor_revision_change_reextracts_before_validating_retained_import_rows() assert_eq!( rebuilt.files[0].extraction.extractor_revision.as_str(), - "extractor.rust.v10" + "extractor.rust.v11" ); assert_ne!( rebuilt.files[0].extraction.parser_import_rows_digest, @@ -425,7 +425,7 @@ fn physical_artifact_reuse_rejects_a_stale_extractor_revision() { assert_eq!( rebuilt.files[0].extraction.extractor_revision.as_str(), - "extractor.rust.v10" + "extractor.rust.v11" ); assert!( rebuilt.files[0] diff --git a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs index 89c3b65fb3..6e7d5f7894 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs @@ -3244,22 +3244,22 @@ fn partitioned_codec_fixture() -> ( } const PARTITIONED_FORMAT_STATE_DIGEST: &str = - "sha256:8d84348830efc4452a078cfac1cc78e0ed44112a37f1025bd6f4f4bc152fe196"; + "sha256:4f78e1a1b0a4ea366f748e4699d4c28d9913782912809bbe2afafba4eac25268"; const PARTITIONED_FORMAT_SEGMENTS: &[(&str, u64)] = &[ ( - "sha256:e50d2733b5f594d79fdccc3e44b5d30d5efb66d14805b5fe0c67d5ceb0a1d66f", + "sha256:7cd13a44df02cc2dc2e13d5a867aaab5410594eaa398fe535ed59450ddc63b36", 11_071, ), ( - "sha256:1095d61bb8bbbf6637f85ca957a510d221aaba7923af8e60b0f3eef07042e6ff", + "sha256:64b5c4d5c08f363d66c1dc3922fb61df7c8df0b022dab6b72507e61e6e1bf403", 5_171, ), ( - "sha256:9921ca7da5c489307887ab570a5e8d5a7cebf192b9b6664c487e9a327943e396", + "sha256:782d1321bdc39aac9efcff85a44f8d48da66e24212d28150020cb04309d23880", 6_279, ), ( - "sha256:52b5707b5312bcb1e29849372b0dbb882205b3289b345643620a35c1d260c246", + "sha256:3a307f49e46059b54a86dac6921287afbe0b25cadc581840b5143ce2bd5a04d1", 6_837, ), ]; From 97288d7acedfd3e8d2a5e36365d6107b07548c82 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 07:36:34 +0000 Subject: [PATCH 49/84] test(daemon): retire an endpoint only after a forced stop The restart journeys reassign the guard: `daemon = spawn_...` evaluates the RHS first, so the successor binds `.tracedecay/daemon.sock` before the predecessor guard is dropped. A predecessor that already exited on SIGTERM had unlinked its own socket, so the unconditional retire in `Drop` removed the *successor's* live path. Every later connect in `ignored_dependency_admission_survives_physical_daemon_restart_without_widening` and `mounted_incremental_lifecycle_preserves_only_complete_compatible_generations` then failed with ENOENT. Retire the recorded endpoint only when this call force-stopped a live child. A child that exited on its own already ran its own cleanup, so there is nothing this guard still owns. Co-Authored-By: Claude Fable 5.1 --- crates/tracedecay/tests/common/mod.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay/tests/common/mod.rs b/crates/tracedecay/tests/common/mod.rs index 624a492e20..2f0774b287 100644 --- a/crates/tracedecay/tests/common/mod.rs +++ b/crates/tracedecay/tests/common/mod.rs @@ -685,8 +685,19 @@ impl TestChildProcess { self.owned_unix_endpoint = Some(path); } - fn retire_owned_unix_endpoint(&mut self) { - if let Some(path) = self.owned_unix_endpoint.take() { + /// Unlink the recorded endpoint, but only after a stop this call forced. + /// + /// A child that exited on its own already ran its own endpoint cleanup, so + /// the path is either gone or has since been rebound by a replacement. The + /// restart journeys reassign the guard (`daemon = spawn_...`), which drops + /// the predecessor *after* the successor has bound the same path; unlinking + /// there would retire a live daemon's socket. + fn retire_owned_unix_endpoint(&mut self, forced: bool) { + let path = self.owned_unix_endpoint.take(); + if !forced { + return; + } + if let Some(path) = path { let _ = std::fs::remove_file(path); } } @@ -779,8 +790,9 @@ impl TestChildProcess { /// primitive elsewhere, keeping fault-injection tests portable. The owned /// Unix endpoint, when recorded, is unlinked only after that reap. pub fn kill_and_wait(&mut self) -> std::io::Result { + let forced = self.is_running(); let status = terminate_and_reap(&mut self.child)?; - self.retire_owned_unix_endpoint(); + self.retire_owned_unix_endpoint(forced); Ok(status) } @@ -805,8 +817,9 @@ impl TestChildProcess { impl Drop for TestChildProcess { fn drop(&mut self) { + let forced = self.is_running(); if terminate_and_reap(&mut self.child).is_ok() { - self.retire_owned_unix_endpoint(); + self.retire_owned_unix_endpoint(forced); } } } From c534d03cf9be71f979bdec597cbf2692f1d16ec4 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 07:37:07 +0000 Subject: [PATCH 50/84] fix(code-index): keep admission ahead of the publication gate The published text projection re-acquired the background admission permit before renaming the active pointer, while `_build_publication` was still held for the rest of the worker iteration. `run_ignored_dependency_admission` takes the same admission *before* that per-worktree gate, so the two orders invert: an ignored-dependency owner holds the permit and waits for the gate, the worker holds the gate and waits for the permit. That stalls the publication pass until the dependency request hits its own deadline and refuses, and with every permit consumed it is a cycle. The repository already states the invariant the re-acquire broke: `background_worker_waits_for_global_admission_before_publication_gate` ("global admission wait must not hold the per-worktree publication gate"). The re-acquire also bought nothing for this PR's claim. `reconcile_pass` is already held across the whole published text projection, so the pointer rename was inside the pass an idle reader samples; dropping the pass across the new permit wait instead opened a fresh window where `reconcile_in_progress` reads zero before the rename has run. Drop the re-acquire and keep the pass guard continuous, which is what makes "idle means the pass tail already ran" true here. The continuation-ordering half of the change is untouched: stamps still land before the pass goes idle. Co-Authored-By: Claude Fable 5.1 --- .../code_index_scheduler/registry/mount.rs | 65 +++++-------------- 1 file changed, 15 insertions(+), 50 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs index f488bde488..b6f9c306d2 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs @@ -1031,13 +1031,15 @@ impl CodeIndexSchedulerRegistryV1 { started_micros, ); } - // Source reconciliation is complete. Release the admission - // permit only across HeadOpening's scheduler-mutex wait: a - // holder of that mutex must be able to run, and an - // ignored-dependency owner still needs this permit before it - // can take the mutex. The publication's text projection - // re-acquires the permit before it renames the active pointer. - // Keep `reconcile_pass` through text seating; dropping it + // Source reconciliation is complete: release the background + // admission permit before HeadOpening / graph work so sibling + // stores can start. The permit is never re-acquired inside + // this pass: `_build_publication` is held for the rest of the + // iteration, and `run_ignored_dependency_admission` takes the + // admission *before* that same gate, so waiting on admission + // here would invert that order (see + // `background_worker_waits_for_global_admission_before_publication_gate`). + // Keep `reconcile_pass` through text seating, dropping it // made `reconcile_in_progress` lie while this worker still // owned graph try_lock, which deadlocked tests that hold the // scheduler mutex and wait for that flag. @@ -1114,49 +1116,12 @@ impl CodeIndexSchedulerRegistryV1 { && !graph_activation_deferred && let Some(text) = graph_text.clone() { - // Head opening released the permit so it could wait on - // the scheduler mutex. Take it back for the pointer - // rename. Drop the pass across that wait: a caller - // holding the permit and waiting for the pass would - // otherwise deadlock, and the pass is re-entered - // before the rename so idle still means the pointer - // write has finished. - let resume_pass = reconcile_pass.is_some(); - drop(reconcile_pass.take()); - let Ok(_text_artifact_admission) = hotpath::future!( - Arc::clone(&worker_background_reconcile_admission).acquire_owned(), - label = "daemon.code_index.admission_wait" - ) - .await - else { - tracing::info!( - event = "code_index_worker_shutdown_observed", - phase = "published_text_projection", - "code-index worker observed shutdown and stopped its pass" - ); - Self::join_retained_text_projection_on_worker_exit( - &mut retained_text_projection, - ) - .await; - return; - }; - if worker_shutting_down.load(Ordering::Acquire) { - tracing::info!( - event = "code_index_worker_shutdown_observed", - phase = "published_text_projection", - "code-index worker observed shutdown and stopped its pass" - ); - Self::join_retained_text_projection_on_worker_exit( - &mut retained_text_projection, - ) - .await; - return; - } - if resume_pass { - reconcile_pass = Some(super::super::ReconcilePassGuard::enter( - &worker_reconcile_in_progress, - )); - } + // `reconcile_pass` is held across this projection, so + // the pointer rename is inside the pass a reader + // samples. Taking the admission permit back here + // instead would deadlock against an + // ignored-dependency owner that already holds it and + // is waiting for `_build_publication`. let projection = tokio::spawn(Self::drive_text_projection( text, Arc::clone(&worker_shutting_down), From 1703387b728d471ab0aedcbd232896f397a8c5a4 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 07:41:22 +0000 Subject: [PATCH 51/84] fix(global-db): write reconciled rebuild sessions set-based The per-row UPDATE loop added by this branch held the activation write transaction open for one executor submission per reconciled session, which blocks admission on a large history (AGENTS.md: rebuilds must not block admission). Write every merged overlap through one json_each upsert instead; the reconcile itself still runs in Rust, so each column is taken verbatim from the merged row. The merge test now carries two overlaps so a mis-correlated batch fails. Co-Authored-By: Claude Fable 5.1 --- .../src/observation_projection/rebuild.rs | 120 ++++++++++++------ 1 file changed, 79 insertions(+), 41 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs index 57076d0a7c..8998fa4c7e 100644 --- a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs +++ b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs @@ -43,6 +43,7 @@ const PROJECTION_RETRY_MAX_MICROS: i64 = 300_000_000; static NEVER_CANCELLED: AtomicBool = AtomicBool::new(false); const SESSION_JSON_COLUMN: &str = "session_json"; +const MERGED_SESSION_JSON_COLUMN: &str = "merged.value"; const MESSAGE_JSON_COLUMN: &str = "message_json"; const STAGED_MESSAGE_JSON_COLUMN: &str = "staged.message_json"; @@ -2221,36 +2222,47 @@ fn decode_overlapping_session(row: &Row) -> ProjectionStoreResult }) } -async fn write_reconciled_session( +/// Write every reconciled overlap in one set-based statement. Rebuild +/// activation owns the database writer, so a per-row `UPDATE` loop would hold +/// admission for as long as the history is large; the merge itself already ran +/// in Rust, so each column is taken verbatim from the merged row. +async fn write_reconciled_sessions( conn: &impl Executor, - merged: &SessionRecord, + merged: &[SessionRecord], ) -> ProjectionStoreResult<()> { + if merged.is_empty() { + return Ok(()); + } + let rows = encode_json(&merged, "encode reconciled projection sessions")?; + let session_extracts = + json_extract_select_list(MERGED_SESSION_JSON_COLUMN, SESSION_JSON_FIELDS); + let assignments = SESSION_JSON_FIELDS + .iter() + .map(|field| format!("{field} = excluded.{field}")) + .collect::>() + .join(",\n "); conn.execute( - "UPDATE sessions - SET project_key = ?3, project_path = ?4, title = ?5, started_at = ?6, - ended_at = ?7, transcript_path = ?8, metadata_json = ?9, - parent_session_id = ?10, is_subagent = ?11, agent_id = ?12, - parent_tool_use_id = ?13 - WHERE provider = ?1 AND session_id = ?2", - params![ - merged.provider.as_str(), - merged.session_id.as_str(), - merged.project_key.as_str(), - merged.project_path.as_str(), - merged.title.as_deref(), - merged.started_at, - merged.ended_at, - merged.transcript_path.as_deref(), - merged.metadata_json.as_deref(), - merged.parent_session_id.as_deref(), - i64::from(merged.is_subagent), - merged.agent_id.as_deref(), - merged.parent_tool_use_id.as_deref(), - ], + &format!( + "INSERT INTO sessions ( + provider, session_id, project_key, project_path, title, started_at, ended_at, + transcript_path, metadata_json, parent_session_id, is_subagent, agent_id, + parent_tool_use_id + ) + SELECT {}, {}, + {session_extracts} + FROM json_each(?1) AS merged + -- `WHERE true` disambiguates the upsert clause from a join constraint. + WHERE true + ON CONFLICT(provider, session_id) DO UPDATE SET + {assignments}", + json_extract_expr(MERGED_SESSION_JSON_COLUMN, "provider"), + json_extract_expr(MERGED_SESSION_JSON_COLUMN, "session_id"), + ), + params![rows.as_str()], ) .await .map(|_| ()) - .map_err(|error| storage("activate reconciled projection session", error)) + .map_err(|error| storage("activate reconciled projection sessions", error)) } /// Classify every staged session that already exists through @@ -2304,10 +2316,7 @@ async fn reconcile_overlapping_rebuild_sessions( } } drop(overlaps); - for merged in &updates { - write_reconciled_session(conn, merged).await?; - } - Ok(()) + write_reconciled_sessions(conn, &updates).await } async fn activate_rebuild_sessions( @@ -2736,30 +2745,43 @@ mod activation_tests { let harness = RegisteredGlobalDbHarness::open("session-collision-merge").await; let active = session(None, None); assert!(harness.registered.upsert_session(&active).await); + let second_active = SessionRecord { + session_id: "session.second".to_owned(), + ..active.clone() + }; + assert!(harness.registered.upsert_session(&second_active).await); let transaction = harness.registered.begin_write_transaction().await.unwrap(); let staged = session(None, Some("Composer session")); stage(&transaction, "generation.session-merge", &staged).await; + // A second overlap keeps the set-based reconciled write honest: each + // merged row must land on its own session, not the first one twice. + let second_staged = SessionRecord { + title: Some("Second composer session".to_owned()), + ..second_active.clone() + }; let fresh = SessionRecord { provider: "cursor".to_owned(), session_id: "session.fresh".to_owned(), title: Some("fresh session".to_owned()), ..active.clone() }; - transaction - .execute( - "INSERT INTO observation_projection_rebuild_sessions ( + for staged in [&second_staged, &fresh] { + transaction + .execute( + "INSERT INTO observation_projection_rebuild_sessions ( projector_version, generation, provider, session_id, session_json ) VALUES (?1, ?2, ?3, ?4, ?5)", - params![ - SESSION_MESSAGE_PROJECTOR_VERSION, - "generation.session-merge", - fresh.provider.as_str(), - fresh.session_id.as_str(), - serde_json::to_string(&fresh).unwrap().as_str(), - ], - ) - .await - .unwrap(); + params![ + SESSION_MESSAGE_PROJECTOR_VERSION, + "generation.session-merge", + staged.provider.as_str(), + staged.session_id.as_str(), + serde_json::to_string(staged).unwrap().as_str(), + ], + ) + .await + .unwrap(); + } activate_rebuild_sessions(&transaction, "generation.session-merge") .await @@ -2781,6 +2803,22 @@ mod activation_tests { .unwrap(); assert_eq!(title, "Composer session"); drop(rows); + let mut rows = transaction + .query( + "SELECT title FROM sessions WHERE provider = 'cursor' AND session_id = 'session.second'", + (), + ) + .await + .unwrap(); + let second_title = rows + .next() + .await + .unwrap() + .unwrap() + .get::(0) + .unwrap(); + assert_eq!(second_title, "Second composer session"); + drop(rows); let mut rows = transaction .query( "SELECT title FROM sessions WHERE provider = 'cursor' AND session_id = 'session.fresh'", From d59b7eba9d3f38ec559bbcb7bd07d6fddfa3d76a Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 07:48:11 +0000 Subject: [PATCH 52/84] test(daemon): never address a process group by a reaped pid `terminate_and_reap` signals `kill(-pid, SIGKILL)` before its own `wait`, and both `kill_and_wait` and `Drop` call it. Every child that a test already reaped - `wait_with_output`, `wait_for_exit`, `is_running` all reap through `try_wait` - therefore got a second group signal from `Drop` on a pid the kernel had already freed, which addresses whatever process group later claims it. Record the reap on `TestChildProcess` and signal the group only while the pid is still ours. A leader that exited but has not been waited on is still an unreaped zombie owning its pid, so the descendant kill the socket proof relies on is unchanged. Co-Authored-By: Claude Fable 5.1 --- crates/tracedecay/tests/common/mod.rs | 32 ++++++++++++++++++++------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay/tests/common/mod.rs b/crates/tracedecay/tests/common/mod.rs index 97f60f8423..46a0985ddd 100644 --- a/crates/tracedecay/tests/common/mod.rs +++ b/crates/tracedecay/tests/common/mod.rs @@ -657,6 +657,9 @@ pub fn http_agent_with_timeout(timeout: Duration) -> ureq::Agent { /// panic while the child is still running, `Drop` force-stops and reaps it. pub struct TestChildProcess { child: Child, + /// Whether this child has been waited on. A reaped pid belongs to the + /// kernel again, so it must never be used to address a process group. + reaped: bool, /// Path of a Unix socket this child published. Released after the process /// group is reaped so a descendant that still holds the listen descriptor /// cannot keep the path accepting. @@ -671,6 +674,7 @@ impl TestChildProcess { pub fn new(child: Child) -> Self { Self { child, + reaped: false, #[cfg(unix)] release_socket: None, } @@ -712,7 +716,9 @@ impl TestChildProcess { } pub fn try_wait(&mut self) -> std::io::Result> { - self.child.try_wait() + let status = self.child.try_wait()?; + self.reaped |= status.is_some(); + Ok(status) } pub fn wait_for_exit(&mut self, timeout: Duration) -> std::io::Result> { @@ -792,10 +798,11 @@ impl TestChildProcess { /// the child's process group is signaled first, then the published socket /// path is unlinked. pub fn kill_and_wait(&mut self) -> std::io::Result { - let status = terminate_and_reap(&mut self.child)?; + let status = terminate_and_reap(&mut self.child, !self.reaped); + self.reaped = true; #[cfg(unix)] self.release_recorded_socket(); - Ok(status) + status } fn drain_stderr(&mut self) { @@ -819,7 +826,8 @@ impl TestChildProcess { impl Drop for TestChildProcess { fn drop(&mut self) { - let _ = terminate_and_reap(&mut self.child); + let _ = terminate_and_reap(&mut self.child, !self.reaped); + self.reaped = true; #[cfg(unix)] self.release_recorded_socket(); } @@ -832,12 +840,20 @@ impl Drop for TestChildProcess { /// inherited, including a listen socket, so the path stays connectable after /// `wait` returns. Signaling the group first closes those descriptors; the /// leader kill still covers a child whose `setpgid` has not run yet. -fn terminate_and_reap(child: &mut Child) -> std::io::Result { +/// +/// `signal_group` must be false once this child has been waited on: a reaped +/// pid is the kernel's to reissue, so negating it could address a process +/// group this harness never created. +fn terminate_and_reap(child: &mut Child, signal_group: bool) -> std::io::Result { // Signal the group before reaping. A leader that has already exited still - // names the group; returning on `try_wait` first would leave descendants - // holding the listen socket. + // names the group while it is an unreaped zombie; returning on `try_wait` + // first would leave descendants holding the listen socket. #[cfg(unix)] - signal_child_process_group(child.id()); + if signal_group { + signal_child_process_group(child.id()); + } + #[cfg(not(unix))] + let _ = signal_group; if let Ok(Some(status)) = child.try_wait() { return Ok(status); From e441823e1744522e6030883e89e02217b125b27b Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 07:54:37 +0000 Subject: [PATCH 53/84] fix(code-index): refuse a sealed generation the checkout moved past Dropping the witness generation-id guard let the graph-on retained reconcile return `Noop` whenever the sealed file digests still matched, including after a commit or branch switch that touches no indexed byte (an empty or docs-only commit). `accept_unchanged_sealed_snapshot` then persisted the new git-metadata sample onto the retained generation's witness, so the stale `reference`/`source_revision` attribution stayed masked until code bytes moved. `finish_retained_reconcile` rebuilds on exactly that drift, and `branch_generations` resolves generations by the commit they sealed, so the retained generation must not outlive it. Gate the accept on the attribution a fresh capture would seal: HEAD's ref must still match the snapshot's, and a snapshot that sealed a revision must still name HEAD's commit. `self.identity` is re-resolved a few lines above, so this adds no walk. A snapshot sealed from a dirty tree carries no revision and keeps the fast path. Verified by `a_moved_commit_refuses_the_sealed_generation_despite_identical_bytes`, which fails on the parent commit with `Some(Noop(..))`. Co-Authored-By: Claude Fable 5.1 --- .../src/code_index_scheduler/reconcile.rs | 20 +++++- .../code_index_scheduler/tests/reconcile.rs | 63 +++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 2ba6cbacf4..b686c00dc6 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -1874,12 +1874,30 @@ impl CodeIndexWorktreeSchedulerV1 { witness.git_metadata_signature == sampled_metadata.stable_signature() && witness.stat_signature == sampled_sweep.signature }); + // Identical source bytes do not make a moved commit or branch the same + // generation. `finish_retained_reconcile` rebuilds on exactly this + // drift, and branch-scoped reads resolve generations by their sealed + // `source_revision`, so accepting here would leave the retained + // generation attributed to a commit the checkout has left for as long + // as the bytes hold still. `self.identity` was re-resolved above, so + // this costs no extra walk. A snapshot sealed without a revision + // (a dirty capture) has no commit attribution to invalidate. + let sealed_attribution_is_current = metadata.snapshot().reference.as_ref() + == self.identity.head_ref() + && metadata + .snapshot() + .source_revision + .as_ref() + .is_none_or(|sealed| self.identity.head_commit() == Some(sealed)); // Graph-on refuses to decode the sealed generation just because the // predecessor witness, or a git-index mtime this seal itself moved, // does not name this generation. The sealed digests are the proof. // Graph-off still captures so a metadata-only drift is verified // without a full decode when the quiet witness is absent. - if sealed_bytes_match && (quiet_witness || !rebuild_changed_source_without_decode) { + if sealed_bytes_match + && sealed_attribution_is_current + && (quiet_witness || !rebuild_changed_source_without_decode) + { return Ok(Some(self.accept_unchanged_sealed_snapshot( metadata, sampled_metadata, diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs index f92911fa98..95eac2b6cd 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs @@ -10495,3 +10495,66 @@ fn serving_swap_seats_a_generation_whose_publication_moved_while_it_activated() "neither refusing arm writes the serving slot" ); } + +/// Unchanged source bytes are not a reason to keep a generation the checkout +/// has committed past. An empty (or docs-only) commit moves HEAD without +/// touching one indexed byte, and `finish_retained_reconcile` rebuilds on +/// exactly that `source_revision` drift because branch-scoped reads resolve +/// generations by the commit they sealed. Accepting the sealed snapshot here +/// would pin the stale attribution for as long as the bytes hold still. +#[test] +fn a_moved_commit_refuses_the_sealed_generation_despite_identical_bytes() { + let fixture = GitFixture::new(ALPHA_LIB_V1); + let store = TempDir::new().expect("store root"); + let mut scheduler = scheduler( + &fixture, + store.path().to_path_buf(), + Arc::new(SharedCodeIndexBytePoolV1::default()), + ); + published(scheduler.reconcile_now().expect("seed retained generation")); + let metadata = scheduler + .servable_retained_text_generation() + .expect("publication store") + .expect("authenticated retained text generation") + .metadata() + .clone(); + let sealed_revision = metadata + .snapshot() + .source_revision + .clone() + .expect("a clean seed seals its commit"); + git( + fixture.path(), + &["commit", "-qm", "docs only", "--allow-empty"], + ); + let moved_head = + CommitId::new(git_stdout(fixture.path(), &["rev-parse", "HEAD"])).expect("moved HEAD"); + assert_ne!(sealed_revision, moved_head, "the fixture must move HEAD"); + + let refused = scheduler + .reconcile_retained_text_generation_with(&metadata, false) + .expect("graph-on retained reconcile"); + assert!( + refused.is_none(), + "a moved commit must not keep the generation sealed at {sealed_revision:?}: {refused:?}" + ); + + // The refusal is what hands the pass to the authoritative capture, and + // that capture is what re-attributes the generation to the new commit. + published( + scheduler + .reconcile_now() + .expect("rebuild at the moved commit"), + ); + assert_eq!( + scheduler + .servable_retained_text_generation() + .expect("publication store") + .expect("authenticated retained text generation") + .metadata() + .snapshot() + .source_revision, + Some(moved_head), + "the rebuilt generation must name the commit the checkout is on" + ); +} From 0201b1364cdc371ac1bf288cebe9a1f5e350aa8d Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 08:03:54 +0000 Subject: [PATCH 54/84] fix(global-db): receipt a reached-frontier replay in this epoch The pre-submit short-circuit returned ExactDuplicate without reaching the writer, so an exact cursor-advance replay no longer registered a receipt under the admitted authority epoch and retained_runtime_ledger_replays_during_bounded_background_convergence found one ledger row instead of two. Keep the frontier check as the flag that maps the outcome, the way the pre-change adapter did, so the reached frontier still reports a duplicate and the replay still receipts. Co-Authored-By: Claude Fable 5.1 --- .../src/observation_adapter.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation_adapter.rs b/crates/tracedecay-global-db/src/observation_adapter.rs index df5263e2b1..965107ef73 100644 --- a/crates/tracedecay-global-db/src/observation_adapter.rs +++ b/crates/tracedecay-global-db/src/observation_adapter.rs @@ -1487,14 +1487,13 @@ impl ObservationStore for GlobalDbObservationStore { )?; // One owner per frontier. A cursor that already reached `next` has // recorded the range; a second reason must not become a permanent - // collision that both ingest owners then warn on forever. - if actual_cursor + // collision that both ingest owners then warn on forever. The + // command still goes to the writer so the current authority epoch + // receipts the replay; only its outcome is reported as a duplicate. + let reached_frontier = actual_cursor .as_ref() - .is_some_and(|cursor| cursor.reached(advance.next_cursor())) - { - return Ok(CursorAdvanceOutcome::ExactDuplicate); - } - if actual_cursor.as_ref() != advance.expected_cursor() { + .is_some_and(|cursor| cursor.reached(advance.next_cursor())); + if !reached_frontier && actual_cursor.as_ref() != advance.expected_cursor() { return Err(ObservationStoreError::CursorConflict { expected: Box::new(advance.expected_cursor().cloned()), actual: Box::new(actual_cursor), @@ -1524,6 +1523,12 @@ impl ObservationStore for GlobalDbObservationStore { ) .await; match outcome? { + RuntimeSubmitOutcomeV1::Committed { .. } + | RuntimeSubmitOutcomeV1::CommittedAfterCancellation { .. } + if reached_frontier => + { + Ok(CursorAdvanceOutcome::ExactDuplicate) + } RuntimeSubmitOutcomeV1::Committed { .. } | RuntimeSubmitOutcomeV1::CommittedAfterCancellation { .. } => { Ok(CursorAdvanceOutcome::Committed) From 8889736b9ac454075da28cbba185eaf0f295903c Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 08:04:31 +0000 Subject: [PATCH 55/84] test(daemon): unlink only the socket this child still publishes The restart journeys reassign their handle - `daemon = spawn_tracedecay_ daemon_with(..)` - so the predecessor is dropped *after* the successor bound a new socket at the same path. Releasing by path alone unlinked the live daemon's endpoint and the next request failed with `daemon_connect_ down ... daemon.sock: No such file or directory`; twelve daemon_suite journeys failed that way, `ignored_dependency_admission_survives_physical_ daemon_restart_without_widening` on every run. Comparing the file's `(dev, ino)` does not separate them: the successor's socket lands on the inode the predecessor's shutdown just freed, which reproduced 5 of 5. Record the publisher instead - claiming a path evicts the previous claim - and unlink only while this child still holds it. Co-Authored-By: Claude Fable 5.1 --- crates/tracedecay/tests/common/mod.rs | 65 ++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 11 deletions(-) diff --git a/crates/tracedecay/tests/common/mod.rs b/crates/tracedecay/tests/common/mod.rs index 46a0985ddd..20b9e5dca0 100644 --- a/crates/tracedecay/tests/common/mod.rs +++ b/crates/tracedecay/tests/common/mod.rs @@ -17,6 +17,8 @@ use std::os::unix::fs::PermissionsExt; use std::os::unix::process::CommandExt; use std::path::{Path, PathBuf}; use std::process::{Child, Command, ExitStatus, Output, Stdio}; +#[cfg(unix)] +use std::sync::{Mutex, PoisonError}; use std::time::{Duration, Instant}; use serde_json::Value; @@ -680,29 +682,41 @@ impl TestChildProcess { } } - /// Unlink `path` once this child has been reaped. + /// Unlink the socket this child published once it has been reaped. /// /// `process_group(0)` makes the child a group leader. Stopping only that /// pid leaves descendants that still hold the listen socket. Group-kill /// closes those descriptors; unlinking the path is what makes a later /// `connect` fail even if the kernel has not finished the last close. + /// + /// Recording claims the path: a restart journey reassigns its handle + /// (`daemon = spawn(..)`), so the successor is already publishing when + /// the predecessor is dropped, and only the current publisher may unlink. + /// File identity is not enough for that - the successor's socket routinely + /// lands on the inode the predecessor's shutdown just freed. #[cfg(unix)] pub fn release_socket_on_stop(&mut self, path: PathBuf) { + claim_published_socket(&path, self.child.id()); self.release_socket = Some(path); } #[cfg(unix)] fn release_recorded_socket(&mut self) { - if let Some(path) = self.release_socket.take() { - match std::fs::remove_file(&path) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - panic!( - "failed to release daemon socket '{}': {error}", - path.display() - ) - } + let Some(path) = self.release_socket.take() else { + return; + }; + if !release_published_socket_claim(&path, self.child.id()) { + // A successor publishes here now; its socket is not ours to unlink. + return; + } + match std::fs::remove_file(&path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + panic!( + "failed to release daemon socket '{}': {error}", + path.display() + ) } } } @@ -869,6 +883,35 @@ fn terminate_and_reap(child: &mut Child, signal_group: bool) -> std::io::Result< child.wait() } +/// The child pid currently publishing each recorded socket path. +#[cfg(unix)] +static PUBLISHED_SOCKETS: Mutex> = Mutex::new(Vec::new()); + +#[cfg(unix)] +fn claim_published_socket(path: &Path, pid: u32) { + let mut claims = PUBLISHED_SOCKETS + .lock() + .unwrap_or_else(PoisonError::into_inner); + claims.retain(|(claimed, _)| claimed != path); + claims.push((path.to_path_buf(), pid)); +} + +/// True when `pid` is still the publisher of `path`, dropping the claim. +#[cfg(unix)] +fn release_published_socket_claim(path: &Path, pid: u32) -> bool { + let mut claims = PUBLISHED_SOCKETS + .lock() + .unwrap_or_else(PoisonError::into_inner); + let Some(index) = claims + .iter() + .position(|(claimed, owner)| claimed == path && *owner == pid) + else { + return false; + }; + claims.swap_remove(index); + true +} + #[cfg(unix)] fn signal_child_process_group(pid: u32) { let Ok(pid) = i32::try_from(pid) else { From 792c4feb956df7390b5393a007beac22e82436a3 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 08:22:00 +0000 Subject: [PATCH 56/84] style(session-temporal): keep the test module last and rustfmt master Master run 35431539771 failed Check formatting (projector.rs, query.rs) and Clippy (items_after_test_module in query.rs) after #1844/#1845 merged without CI. Co-Authored-By: Claude Fable 5.1 --- .../projector.rs | 8 ++--- .../src/query.rs | 29 +++++++++---------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs index 10818733c1..632898ca24 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs @@ -119,11 +119,11 @@ impl SessionTemporalRefreshProjector for CanonicalSessionTemporalProjector { // Empty remaining range is a durable no-op: terminalize with an // empty complete progress batch instead of deferring forever. Ok(None) => canonical_noop_complete_effect(&recovery), - Err(error) if error.is_storage() => Err( - SessionTemporalRefreshProjectorError::retryable(format!( + Err(error) if error.is_storage() => { + Err(SessionTemporalRefreshProjectorError::retryable(format!( "source_busy: {error}" - )), - ), + ))) + } Err(_) => Err(SessionTemporalRefreshProjectorError::terminal( "projector_failed", )), diff --git a/crates/tracedecay-session-temporal-store/src/query.rs b/crates/tracedecay-session-temporal-store/src/query.rs index 367b14bb79..ae0ab6b4de 100644 --- a/crates/tracedecay-session-temporal-store/src/query.rs +++ b/crates/tracedecay-session-temporal-store/src/query.rs @@ -246,8 +246,7 @@ pub(super) async fn read_observations( // ceiling. Split until a single observation remains; that // observation is then a typed storage failure, not a retry // that looks like a busy source. - if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 - { + if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 { let mid = start + chunk.len() / 2; pending.push((mid, end)); pending.push((start, mid)); @@ -289,11 +288,18 @@ fn observation_prefetch_exceeded_materialization_limit(error: &SessionStoreError } } +/// The error `read_observation` raises for an id the store does not hold, reused +/// by callers that resolve prefetched observations out of a batch map. +pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { + storage_message( + PERSIST_OPERATION, + format!("source observation {} is missing", observation_id.as_str()), + ) +} + #[cfg(test)] mod tests { - use super::{ - PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage, - }; + use super::{PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage}; #[test] fn materialization_limit_is_the_prefetch_split_signal() { @@ -308,15 +314,8 @@ mod tests { assert!(observation_prefetch_exceeded_materialization_limit( &exceeded )); - assert!(!observation_prefetch_exceeded_materialization_limit(&locked)); + assert!(!observation_prefetch_exceeded_materialization_limit( + &locked + )); } } - -/// The error `read_observation` raises for an id the store does not hold, reused -/// by callers that resolve prefetched observations out of a batch map. -pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { - storage_message( - PERSIST_OPERATION, - format!("source observation {} is missing", observation_id.as_str()), - ) -} From 5ccb3c19915a2b8e00a3bec8b74bd3c64e5af9ab Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 08:22:00 +0000 Subject: [PATCH 57/84] style(session-temporal): keep the test module last and rustfmt master Master run 35431539771 failed Check formatting (projector.rs, query.rs) and Clippy (items_after_test_module in query.rs) after #1844/#1845 merged without CI. Co-Authored-By: Claude Fable 5.1 --- .../projector.rs | 8 ++--- .../src/query.rs | 29 +++++++++---------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs index 10818733c1..632898ca24 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs @@ -119,11 +119,11 @@ impl SessionTemporalRefreshProjector for CanonicalSessionTemporalProjector { // Empty remaining range is a durable no-op: terminalize with an // empty complete progress batch instead of deferring forever. Ok(None) => canonical_noop_complete_effect(&recovery), - Err(error) if error.is_storage() => Err( - SessionTemporalRefreshProjectorError::retryable(format!( + Err(error) if error.is_storage() => { + Err(SessionTemporalRefreshProjectorError::retryable(format!( "source_busy: {error}" - )), - ), + ))) + } Err(_) => Err(SessionTemporalRefreshProjectorError::terminal( "projector_failed", )), diff --git a/crates/tracedecay-session-temporal-store/src/query.rs b/crates/tracedecay-session-temporal-store/src/query.rs index 367b14bb79..ae0ab6b4de 100644 --- a/crates/tracedecay-session-temporal-store/src/query.rs +++ b/crates/tracedecay-session-temporal-store/src/query.rs @@ -246,8 +246,7 @@ pub(super) async fn read_observations( // ceiling. Split until a single observation remains; that // observation is then a typed storage failure, not a retry // that looks like a busy source. - if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 - { + if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 { let mid = start + chunk.len() / 2; pending.push((mid, end)); pending.push((start, mid)); @@ -289,11 +288,18 @@ fn observation_prefetch_exceeded_materialization_limit(error: &SessionStoreError } } +/// The error `read_observation` raises for an id the store does not hold, reused +/// by callers that resolve prefetched observations out of a batch map. +pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { + storage_message( + PERSIST_OPERATION, + format!("source observation {} is missing", observation_id.as_str()), + ) +} + #[cfg(test)] mod tests { - use super::{ - PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage, - }; + use super::{PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage}; #[test] fn materialization_limit_is_the_prefetch_split_signal() { @@ -308,15 +314,8 @@ mod tests { assert!(observation_prefetch_exceeded_materialization_limit( &exceeded )); - assert!(!observation_prefetch_exceeded_materialization_limit(&locked)); + assert!(!observation_prefetch_exceeded_materialization_limit( + &locked + )); } } - -/// The error `read_observation` raises for an id the store does not hold, reused -/// by callers that resolve prefetched observations out of a batch map. -pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { - storage_message( - PERSIST_OPERATION, - format!("source observation {} is missing", observation_id.as_str()), - ) -} From 99b7bca2744b92fb2d6d988400dc422e947fd5d9 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 08:22:00 +0000 Subject: [PATCH 58/84] style(session-temporal): keep the test module last and rustfmt master Master run 35431539771 failed Check formatting (projector.rs, query.rs) and Clippy (items_after_test_module in query.rs) after #1844/#1845 merged without CI. Co-Authored-By: Claude Fable 5.1 --- .../projector.rs | 8 ++--- .../src/query.rs | 29 +++++++++---------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs index 10818733c1..632898ca24 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs @@ -119,11 +119,11 @@ impl SessionTemporalRefreshProjector for CanonicalSessionTemporalProjector { // Empty remaining range is a durable no-op: terminalize with an // empty complete progress batch instead of deferring forever. Ok(None) => canonical_noop_complete_effect(&recovery), - Err(error) if error.is_storage() => Err( - SessionTemporalRefreshProjectorError::retryable(format!( + Err(error) if error.is_storage() => { + Err(SessionTemporalRefreshProjectorError::retryable(format!( "source_busy: {error}" - )), - ), + ))) + } Err(_) => Err(SessionTemporalRefreshProjectorError::terminal( "projector_failed", )), diff --git a/crates/tracedecay-session-temporal-store/src/query.rs b/crates/tracedecay-session-temporal-store/src/query.rs index 367b14bb79..ae0ab6b4de 100644 --- a/crates/tracedecay-session-temporal-store/src/query.rs +++ b/crates/tracedecay-session-temporal-store/src/query.rs @@ -246,8 +246,7 @@ pub(super) async fn read_observations( // ceiling. Split until a single observation remains; that // observation is then a typed storage failure, not a retry // that looks like a busy source. - if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 - { + if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 { let mid = start + chunk.len() / 2; pending.push((mid, end)); pending.push((start, mid)); @@ -289,11 +288,18 @@ fn observation_prefetch_exceeded_materialization_limit(error: &SessionStoreError } } +/// The error `read_observation` raises for an id the store does not hold, reused +/// by callers that resolve prefetched observations out of a batch map. +pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { + storage_message( + PERSIST_OPERATION, + format!("source observation {} is missing", observation_id.as_str()), + ) +} + #[cfg(test)] mod tests { - use super::{ - PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage, - }; + use super::{PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage}; #[test] fn materialization_limit_is_the_prefetch_split_signal() { @@ -308,15 +314,8 @@ mod tests { assert!(observation_prefetch_exceeded_materialization_limit( &exceeded )); - assert!(!observation_prefetch_exceeded_materialization_limit(&locked)); + assert!(!observation_prefetch_exceeded_materialization_limit( + &locked + )); } } - -/// The error `read_observation` raises for an id the store does not hold, reused -/// by callers that resolve prefetched observations out of a batch map. -pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { - storage_message( - PERSIST_OPERATION, - format!("source observation {} is missing", observation_id.as_str()), - ) -} From e436ba05689b3b08f5c8610d4f1b77e56b62efeb Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 08:34:33 +0000 Subject: [PATCH 59/84] test(code-index): assert the clone occurrence index backfill The resumed-page assertion this branch added passes with or without f2ab8080cf's ensure_clone_occurrence_indexes: master's page reads are plain `WHERE symbol_occurrence_id = ?1` with no INDEXED BY, so a successor staged before the indexes existed returns the same rows by full scan. Commenting the backfill out left the test green. Read sqlite_master after the resume and require both indexes, so the backfill f2ab8080cf claims for copied priors is what fails if it is removed. With the backfill disabled this now fails 0 != 2. Co-Authored-By: Claude Fable 5.1 --- .../search_quality_suite/candidate_producers.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs index 07463f7adf..9009f6ecdd 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs @@ -1470,6 +1470,20 @@ fn v16_clone_payloads_are_content_addressed_and_postings_page() { CODE_LEXICAL_ARTIFACT_BUILD_MEMORY_BUDGET_BYTES_V1, ) .expect("resume clone-only successor"); + // The resume reads the same rows with or without the indexes, so assert + // the backfill itself as well as the verification it is there to speed up. + let backfilled = rusqlite::Connection::open(&successor_path) + .expect("open successor after index backfill") + .query_row( + "SELECT count(*) FROM sqlite_master WHERE type = 'index' AND name IN ('clone_exact_postings_by_occurrence', 'clone_fingerprint_postings_by_occurrence')", + [], + |row| row.get::<_, i64>(0), + ) + .expect("count occurrence indexes"); + assert_eq!( + backfilled, 2, + "opening a successor staged before the occurrence indexes must install both" + ); successor .verify_resumed_page(&pages[0], &control) .expect("resumed clone page verifies through the occurrence index"); From d6d8665a80068bf8b28a995cdff6c412997720be Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 08:34:45 +0000 Subject: [PATCH 60/84] fix(retention): keep absent durable state a storage failure The merged branch reclassified every `NotFound` in the generation census as `GenerationStoreBusy`. Two of those sites are not publisher races: - The generations directory read under a live active pointer. The sealer writes the sealed file, its directory and the pointer under one store-lock hold, so a durable pointer implies a durable directory; an absent directory beside a live pointer is loss. Production consumes `GenerationStoreBusy` with `defer_generation_store_busy`, so deferring here turns that loss into a silent per-tick defer that never reclaims and never reports degraded. - `read_active_pointer`. Its lock-free caller `read_optional_active_pointer` already stats the pointer and owns the typed unpublished answer, and the pointer is installed by atomic rename so it is never transiently absent; its other caller, `mutate_verified_text_artifact_under_lock`, holds the store lock and compares against an expected pointer. The enumerate-then-open sites keep the deferral: the census lists a directory without the store lock, so a name it just read can be unlinked before the open, which is the race the branch set out to fix. Drops the unit test that asserted the reverted classification. Co-Authored-By: Claude Fable 5.1 --- .../src/code_index_generations.rs | 7 +++++-- .../src/code_index_generations/tests.rs | 18 ------------------ 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations.rs b/crates/tracedecay-code-index-retention/src/code_index_generations.rs index 78c68296bd..b5f76713c9 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations.rs @@ -885,7 +885,10 @@ fn plan_code_generation_retention_with_verification_cancellable( Err(error) if error.kind() == std::io::ErrorKind::NotFound && active_pointer.is_none() => { None } - Err(error) => return Err(deferred_if_absent(error)), + // A pointer is only durable once its generation directory is, so a + // live pointer over an absent directory is loss, not a publisher + // race, and must stay loud. + Err(error) => return Err(storage(error)), }; let mut generations = BTreeMap::new(); let mut active_state_digest = None; @@ -1745,7 +1748,7 @@ fn read_active_pointer( store_root: &Path, ) -> Result { let path = store_root.join(ACTIVE_POINTER_FILE); - let bytes = std::fs::read(&path).map_err(deferred_if_absent)?; + let bytes = std::fs::read(&path).map_err(storage)?; serde_json::from_slice(&bytes).map_err(|error| { CodeGenerationRetentionErrorV1::UnsafeState(format!( "active pointer '{}' is corrupt: {error}", diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs index 87baa7ad01..47d6d406f9 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs @@ -1564,24 +1564,6 @@ fn preparation_defers_when_the_scope_root_does_not_exist_yet() { ); } -#[test] -fn preparation_defers_when_the_pointer_exists_without_its_generation_directory() { - let (store, _generations) = fixture_store(1); - std::fs::remove_dir_all(store.path().join(GENERATIONS_DIRECTORY)) - .expect("remove generation directory under a live pointer"); - let error = prepare_next_code_generation_retention_cancellable( - store.path(), - &BTreeSet::new(), - &|| false, - None, - ) - .expect_err("the generation directory is not durable yet"); - assert!( - matches!(error, CodeGenerationRetentionErrorV1::GenerationStoreBusy), - "a pointer without its generation directory is a torn publish, not a storage failure: {error:?}" - ); -} - #[test] fn metadata_only_segment_census_observes_at_most_one_directory_entry() { let store = tempfile::TempDir::new().expect("create unpublished store"); From b5231fd649060b542b99f601080fb51aeae57e88 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 08:22:00 +0000 Subject: [PATCH 61/84] style(session-temporal): keep the test module last and rustfmt master Master run 35431539771 failed Check formatting (projector.rs, query.rs) and Clippy (items_after_test_module in query.rs) after #1844/#1845 merged without CI. Co-Authored-By: Claude Fable 5.1 --- .../projector.rs | 8 ++--- .../src/query.rs | 29 +++++++++---------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs index 10818733c1..632898ca24 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs @@ -119,11 +119,11 @@ impl SessionTemporalRefreshProjector for CanonicalSessionTemporalProjector { // Empty remaining range is a durable no-op: terminalize with an // empty complete progress batch instead of deferring forever. Ok(None) => canonical_noop_complete_effect(&recovery), - Err(error) if error.is_storage() => Err( - SessionTemporalRefreshProjectorError::retryable(format!( + Err(error) if error.is_storage() => { + Err(SessionTemporalRefreshProjectorError::retryable(format!( "source_busy: {error}" - )), - ), + ))) + } Err(_) => Err(SessionTemporalRefreshProjectorError::terminal( "projector_failed", )), diff --git a/crates/tracedecay-session-temporal-store/src/query.rs b/crates/tracedecay-session-temporal-store/src/query.rs index 367b14bb79..ae0ab6b4de 100644 --- a/crates/tracedecay-session-temporal-store/src/query.rs +++ b/crates/tracedecay-session-temporal-store/src/query.rs @@ -246,8 +246,7 @@ pub(super) async fn read_observations( // ceiling. Split until a single observation remains; that // observation is then a typed storage failure, not a retry // that looks like a busy source. - if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 - { + if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 { let mid = start + chunk.len() / 2; pending.push((mid, end)); pending.push((start, mid)); @@ -289,11 +288,18 @@ fn observation_prefetch_exceeded_materialization_limit(error: &SessionStoreError } } +/// The error `read_observation` raises for an id the store does not hold, reused +/// by callers that resolve prefetched observations out of a batch map. +pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { + storage_message( + PERSIST_OPERATION, + format!("source observation {} is missing", observation_id.as_str()), + ) +} + #[cfg(test)] mod tests { - use super::{ - PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage, - }; + use super::{PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage}; #[test] fn materialization_limit_is_the_prefetch_split_signal() { @@ -308,15 +314,8 @@ mod tests { assert!(observation_prefetch_exceeded_materialization_limit( &exceeded )); - assert!(!observation_prefetch_exceeded_materialization_limit(&locked)); + assert!(!observation_prefetch_exceeded_materialization_limit( + &locked + )); } } - -/// The error `read_observation` raises for an id the store does not hold, reused -/// by callers that resolve prefetched observations out of a batch map. -pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { - storage_message( - PERSIST_OPERATION, - format!("source observation {} is missing", observation_id.as_str()), - ) -} From 8ba2ece6ba0e211f10e23026cfac67f5da2ca6ed Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 08:22:00 +0000 Subject: [PATCH 62/84] style(session-temporal): keep the test module last and rustfmt master Master run 35431539771 failed Check formatting (projector.rs, query.rs) and Clippy (items_after_test_module in query.rs) after #1844/#1845 merged without CI. Co-Authored-By: Claude Fable 5.1 --- .../projector.rs | 8 ++--- .../src/query.rs | 29 +++++++++---------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs index 10818733c1..632898ca24 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs @@ -119,11 +119,11 @@ impl SessionTemporalRefreshProjector for CanonicalSessionTemporalProjector { // Empty remaining range is a durable no-op: terminalize with an // empty complete progress batch instead of deferring forever. Ok(None) => canonical_noop_complete_effect(&recovery), - Err(error) if error.is_storage() => Err( - SessionTemporalRefreshProjectorError::retryable(format!( + Err(error) if error.is_storage() => { + Err(SessionTemporalRefreshProjectorError::retryable(format!( "source_busy: {error}" - )), - ), + ))) + } Err(_) => Err(SessionTemporalRefreshProjectorError::terminal( "projector_failed", )), diff --git a/crates/tracedecay-session-temporal-store/src/query.rs b/crates/tracedecay-session-temporal-store/src/query.rs index 367b14bb79..ae0ab6b4de 100644 --- a/crates/tracedecay-session-temporal-store/src/query.rs +++ b/crates/tracedecay-session-temporal-store/src/query.rs @@ -246,8 +246,7 @@ pub(super) async fn read_observations( // ceiling. Split until a single observation remains; that // observation is then a typed storage failure, not a retry // that looks like a busy source. - if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 - { + if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 { let mid = start + chunk.len() / 2; pending.push((mid, end)); pending.push((start, mid)); @@ -289,11 +288,18 @@ fn observation_prefetch_exceeded_materialization_limit(error: &SessionStoreError } } +/// The error `read_observation` raises for an id the store does not hold, reused +/// by callers that resolve prefetched observations out of a batch map. +pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { + storage_message( + PERSIST_OPERATION, + format!("source observation {} is missing", observation_id.as_str()), + ) +} + #[cfg(test)] mod tests { - use super::{ - PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage, - }; + use super::{PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage}; #[test] fn materialization_limit_is_the_prefetch_split_signal() { @@ -308,15 +314,8 @@ mod tests { assert!(observation_prefetch_exceeded_materialization_limit( &exceeded )); - assert!(!observation_prefetch_exceeded_materialization_limit(&locked)); + assert!(!observation_prefetch_exceeded_materialization_limit( + &locked + )); } } - -/// The error `read_observation` raises for an id the store does not hold, reused -/// by callers that resolve prefetched observations out of a batch map. -pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { - storage_message( - PERSIST_OPERATION, - format!("source observation {} is missing", observation_id.as_str()), - ) -} From 569660d52d72510360e08398afe130d52d560b30 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 08:22:00 +0000 Subject: [PATCH 63/84] style(session-temporal): keep the test module last and rustfmt master Master run 35431539771 failed Check formatting (projector.rs, query.rs) and Clippy (items_after_test_module in query.rs) after #1844/#1845 merged without CI. Co-Authored-By: Claude Fable 5.1 --- .../projector.rs | 8 ++--- .../src/query.rs | 29 +++++++++---------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs index 10818733c1..632898ca24 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs @@ -119,11 +119,11 @@ impl SessionTemporalRefreshProjector for CanonicalSessionTemporalProjector { // Empty remaining range is a durable no-op: terminalize with an // empty complete progress batch instead of deferring forever. Ok(None) => canonical_noop_complete_effect(&recovery), - Err(error) if error.is_storage() => Err( - SessionTemporalRefreshProjectorError::retryable(format!( + Err(error) if error.is_storage() => { + Err(SessionTemporalRefreshProjectorError::retryable(format!( "source_busy: {error}" - )), - ), + ))) + } Err(_) => Err(SessionTemporalRefreshProjectorError::terminal( "projector_failed", )), diff --git a/crates/tracedecay-session-temporal-store/src/query.rs b/crates/tracedecay-session-temporal-store/src/query.rs index 367b14bb79..ae0ab6b4de 100644 --- a/crates/tracedecay-session-temporal-store/src/query.rs +++ b/crates/tracedecay-session-temporal-store/src/query.rs @@ -246,8 +246,7 @@ pub(super) async fn read_observations( // ceiling. Split until a single observation remains; that // observation is then a typed storage failure, not a retry // that looks like a busy source. - if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 - { + if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 { let mid = start + chunk.len() / 2; pending.push((mid, end)); pending.push((start, mid)); @@ -289,11 +288,18 @@ fn observation_prefetch_exceeded_materialization_limit(error: &SessionStoreError } } +/// The error `read_observation` raises for an id the store does not hold, reused +/// by callers that resolve prefetched observations out of a batch map. +pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { + storage_message( + PERSIST_OPERATION, + format!("source observation {} is missing", observation_id.as_str()), + ) +} + #[cfg(test)] mod tests { - use super::{ - PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage, - }; + use super::{PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage}; #[test] fn materialization_limit_is_the_prefetch_split_signal() { @@ -308,15 +314,8 @@ mod tests { assert!(observation_prefetch_exceeded_materialization_limit( &exceeded )); - assert!(!observation_prefetch_exceeded_materialization_limit(&locked)); + assert!(!observation_prefetch_exceeded_materialization_limit( + &locked + )); } } - -/// The error `read_observation` raises for an id the store does not hold, reused -/// by callers that resolve prefetched observations out of a batch map. -pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { - storage_message( - PERSIST_OPERATION, - format!("source observation {} is missing", observation_id.as_str()), - ) -} From e6acc47919a985fb1eed854769080ccb78ede5d9 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 08:47:53 +0000 Subject: [PATCH 64/84] docs(global-db): restore the raw upsert doc split by the adopt helper adopt_owned_projection_raw_session was inserted between upsert_projected_raw_message and its doc comment, so the sanitization-refusal note described a plain session UPDATE and the upsert lost its documentation. No behavior change. Co-Authored-By: Claude Fable 5.1 --- .../src/observation_projection/apply.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation_projection/apply.rs b/crates/tracedecay-global-db/src/observation_projection/apply.rs index 58026795d8..811ee1e92e 100644 --- a/crates/tracedecay-global-db/src/observation_projection/apply.rs +++ b/crates/tracedecay-global-db/src/observation_projection/apply.rs @@ -633,13 +633,6 @@ pub(super) async fn apply_session( } } -/// Writes the projection-derived raw row through the canonical LCM raw -/// authority so it carries the content-bound sanitization receipt that -/// hydration requires; a receipt-less raw row is unreadable, not raw storage. -/// -/// A deterministic sanitization refusal keeps its typed class: mapping it to -/// `Storage` would schedule an endless environmental retry for content that -/// can never succeed, permanently poisoning the sequential projection queue. /// Aligns a provenance-owned raw twin onto the projection's session before /// the content upsert. /// @@ -668,6 +661,13 @@ async fn adopt_owned_projection_raw_session( .map_err(|error| storage("adopt projection raw session", error)) } +/// Writes the projection-derived raw row through the canonical LCM raw +/// authority so it carries the content-bound sanitization receipt that +/// hydration requires; a receipt-less raw row is unreadable, not raw storage. +/// +/// A deterministic sanitization refusal keeps its typed class: mapping it to +/// `Storage` would schedule an endless environmental retry for content that +/// can never succeed, permanently poisoning the sequential projection queue. async fn upsert_projected_raw_message( conn: &impl Executor, message: &SessionMessageRecord, From b6cd13c0b37bd39f0bee48394879551882487dbd Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 08:22:00 +0000 Subject: [PATCH 65/84] style(session-temporal): keep the test module last and rustfmt master Master run 35431539771 failed Check formatting (projector.rs, query.rs) and Clippy (items_after_test_module in query.rs) after #1844/#1845 merged without CI. Co-Authored-By: Claude Fable 5.1 --- .../projector.rs | 8 ++--- .../src/query.rs | 29 +++++++++---------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs index 10818733c1..632898ca24 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs @@ -119,11 +119,11 @@ impl SessionTemporalRefreshProjector for CanonicalSessionTemporalProjector { // Empty remaining range is a durable no-op: terminalize with an // empty complete progress batch instead of deferring forever. Ok(None) => canonical_noop_complete_effect(&recovery), - Err(error) if error.is_storage() => Err( - SessionTemporalRefreshProjectorError::retryable(format!( + Err(error) if error.is_storage() => { + Err(SessionTemporalRefreshProjectorError::retryable(format!( "source_busy: {error}" - )), - ), + ))) + } Err(_) => Err(SessionTemporalRefreshProjectorError::terminal( "projector_failed", )), diff --git a/crates/tracedecay-session-temporal-store/src/query.rs b/crates/tracedecay-session-temporal-store/src/query.rs index 367b14bb79..ae0ab6b4de 100644 --- a/crates/tracedecay-session-temporal-store/src/query.rs +++ b/crates/tracedecay-session-temporal-store/src/query.rs @@ -246,8 +246,7 @@ pub(super) async fn read_observations( // ceiling. Split until a single observation remains; that // observation is then a typed storage failure, not a retry // that looks like a busy source. - if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 - { + if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 { let mid = start + chunk.len() / 2; pending.push((mid, end)); pending.push((start, mid)); @@ -289,11 +288,18 @@ fn observation_prefetch_exceeded_materialization_limit(error: &SessionStoreError } } +/// The error `read_observation` raises for an id the store does not hold, reused +/// by callers that resolve prefetched observations out of a batch map. +pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { + storage_message( + PERSIST_OPERATION, + format!("source observation {} is missing", observation_id.as_str()), + ) +} + #[cfg(test)] mod tests { - use super::{ - PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage, - }; + use super::{PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage}; #[test] fn materialization_limit_is_the_prefetch_split_signal() { @@ -308,15 +314,8 @@ mod tests { assert!(observation_prefetch_exceeded_materialization_limit( &exceeded )); - assert!(!observation_prefetch_exceeded_materialization_limit(&locked)); + assert!(!observation_prefetch_exceeded_materialization_limit( + &locked + )); } } - -/// The error `read_observation` raises for an id the store does not hold, reused -/// by callers that resolve prefetched observations out of a batch map. -pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { - storage_message( - PERSIST_OPERATION, - format!("source observation {} is missing", observation_id.as_str()), - ) -} From fb58663dd361ba11b0e933d267c717fa8244b584 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 08:57:02 +0000 Subject: [PATCH 66/84] fix(cli): use ? for the missing install-probe binary `cargo clippy --workspace --all-targets --locked -- -D warnings` failed with clippy::question_mark on `probed_installed_version`'s `let...else`, which only returned `None`. Behaviour is unchanged. Co-Authored-By: Claude Fable 5.1 --- crates/tracedecay-cli/src/upgrade.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/tracedecay-cli/src/upgrade.rs b/crates/tracedecay-cli/src/upgrade.rs index b93064d147..da334e24d1 100644 --- a/crates/tracedecay-cli/src/upgrade.rs +++ b/crates/tracedecay-cli/src/upgrade.rs @@ -987,10 +987,7 @@ fn installed_binary_version_within( /// commit the binary and the daemon both name, and readiness treats that /// omission as a different identity. fn probed_installed_version(binary: Option<&Path>, owner: &str) -> Option { - let Some(path) = binary else { - return None; - }; - match installed_binary_version(path) { + match installed_binary_version(binary?) { Ok(version) => Some(version), Err(reason) => { eprintln!( From f2831bf2e261cb4d8394a337322df2e987b213a4 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 08:22:00 +0000 Subject: [PATCH 67/84] style(session-temporal): keep the test module last and rustfmt master Master run 35431539771 failed Check formatting (projector.rs, query.rs) and Clippy (items_after_test_module in query.rs) after #1844/#1845 merged without CI. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 0096ead0e30f33aaf7d0dd430d5cd9824892edcb) --- .../projector.rs | 8 ++--- .../src/query.rs | 29 +++++++++---------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs index 10818733c1..632898ca24 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs @@ -119,11 +119,11 @@ impl SessionTemporalRefreshProjector for CanonicalSessionTemporalProjector { // Empty remaining range is a durable no-op: terminalize with an // empty complete progress batch instead of deferring forever. Ok(None) => canonical_noop_complete_effect(&recovery), - Err(error) if error.is_storage() => Err( - SessionTemporalRefreshProjectorError::retryable(format!( + Err(error) if error.is_storage() => { + Err(SessionTemporalRefreshProjectorError::retryable(format!( "source_busy: {error}" - )), - ), + ))) + } Err(_) => Err(SessionTemporalRefreshProjectorError::terminal( "projector_failed", )), diff --git a/crates/tracedecay-session-temporal-store/src/query.rs b/crates/tracedecay-session-temporal-store/src/query.rs index 367b14bb79..ae0ab6b4de 100644 --- a/crates/tracedecay-session-temporal-store/src/query.rs +++ b/crates/tracedecay-session-temporal-store/src/query.rs @@ -246,8 +246,7 @@ pub(super) async fn read_observations( // ceiling. Split until a single observation remains; that // observation is then a typed storage failure, not a retry // that looks like a busy source. - if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 - { + if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 { let mid = start + chunk.len() / 2; pending.push((mid, end)); pending.push((start, mid)); @@ -289,11 +288,18 @@ fn observation_prefetch_exceeded_materialization_limit(error: &SessionStoreError } } +/// The error `read_observation` raises for an id the store does not hold, reused +/// by callers that resolve prefetched observations out of a batch map. +pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { + storage_message( + PERSIST_OPERATION, + format!("source observation {} is missing", observation_id.as_str()), + ) +} + #[cfg(test)] mod tests { - use super::{ - PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage, - }; + use super::{PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage}; #[test] fn materialization_limit_is_the_prefetch_split_signal() { @@ -308,15 +314,8 @@ mod tests { assert!(observation_prefetch_exceeded_materialization_limit( &exceeded )); - assert!(!observation_prefetch_exceeded_materialization_limit(&locked)); + assert!(!observation_prefetch_exceeded_materialization_limit( + &locked + )); } } - -/// The error `read_observation` raises for an id the store does not hold, reused -/// by callers that resolve prefetched observations out of a batch map. -pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { - storage_message( - PERSIST_OPERATION, - format!("source observation {} is missing", observation_id.as_str()), - ) -} From 7458148f70d4473dc929757389526c7b6b4cf2ad Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 08:22:00 +0000 Subject: [PATCH 68/84] style(session-temporal): keep the test module last and rustfmt master Master run 35431539771 failed Check formatting (projector.rs, query.rs) and Clippy (items_after_test_module in query.rs) after #1844/#1845 merged without CI. Co-Authored-By: Claude Fable 5.1 --- .../projector.rs | 8 ++--- .../src/query.rs | 29 +++++++++---------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs index 10818733c1..632898ca24 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs @@ -119,11 +119,11 @@ impl SessionTemporalRefreshProjector for CanonicalSessionTemporalProjector { // Empty remaining range is a durable no-op: terminalize with an // empty complete progress batch instead of deferring forever. Ok(None) => canonical_noop_complete_effect(&recovery), - Err(error) if error.is_storage() => Err( - SessionTemporalRefreshProjectorError::retryable(format!( + Err(error) if error.is_storage() => { + Err(SessionTemporalRefreshProjectorError::retryable(format!( "source_busy: {error}" - )), - ), + ))) + } Err(_) => Err(SessionTemporalRefreshProjectorError::terminal( "projector_failed", )), diff --git a/crates/tracedecay-session-temporal-store/src/query.rs b/crates/tracedecay-session-temporal-store/src/query.rs index 367b14bb79..ae0ab6b4de 100644 --- a/crates/tracedecay-session-temporal-store/src/query.rs +++ b/crates/tracedecay-session-temporal-store/src/query.rs @@ -246,8 +246,7 @@ pub(super) async fn read_observations( // ceiling. Split until a single observation remains; that // observation is then a typed storage failure, not a retry // that looks like a busy source. - if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 - { + if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 { let mid = start + chunk.len() / 2; pending.push((mid, end)); pending.push((start, mid)); @@ -289,11 +288,18 @@ fn observation_prefetch_exceeded_materialization_limit(error: &SessionStoreError } } +/// The error `read_observation` raises for an id the store does not hold, reused +/// by callers that resolve prefetched observations out of a batch map. +pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { + storage_message( + PERSIST_OPERATION, + format!("source observation {} is missing", observation_id.as_str()), + ) +} + #[cfg(test)] mod tests { - use super::{ - PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage, - }; + use super::{PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage}; #[test] fn materialization_limit_is_the_prefetch_split_signal() { @@ -308,15 +314,8 @@ mod tests { assert!(observation_prefetch_exceeded_materialization_limit( &exceeded )); - assert!(!observation_prefetch_exceeded_materialization_limit(&locked)); + assert!(!observation_prefetch_exceeded_materialization_limit( + &locked + )); } } - -/// The error `read_observation` raises for an id the store does not hold, reused -/// by callers that resolve prefetched observations out of a batch map. -pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { - storage_message( - PERSIST_OPERATION, - format!("source observation {} is missing", observation_id.as_str()), - ) -} From e6f94cdb90e3f094a5e14d6e3105fca6a0b11c6f Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 09:24:03 +0000 Subject: [PATCH 69/84] fix(daemon): retry a revoked response in the one-shot client The daemon answers a tool call that lands on a project-server retirement with the typed, retryable project_server_response_revoked error (added for journey clients in 2f0f8d0e4b). The one-shot client behind 'tracedecay tool' keyed its retry only on the project-open subset (warming, deferred discovery, capacity), so the retirement window surfaced as a hard error. Run 35431840590 root-journeys: observation_authority_reset_recovers_the_retained_temporal_authority TRY 1 FAIL with that error from tracedecay_lcm_describe, TRY 2 PASS. is_project_open_retryable_error now also honours tool_call_transport_error_is_retryable, so the client re-sends on its existing cadence and deadline; the unit test pins that a revoked error yields a retry wait. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 0ebaf81524448db53a32cc9318fd4615378f0506) --- crates/tracedecay/src/daemon/core_client.rs | 23 +++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay/src/daemon/core_client.rs b/crates/tracedecay/src/daemon/core_client.rs index c85a8df3bb..a81110e2b2 100644 --- a/crates/tracedecay/src/daemon/core_client.rs +++ b/crates/tracedecay/src/daemon/core_client.rs @@ -23,7 +23,7 @@ use super::unavailable_error; use super::{ BrokerStream, DaemonAuthPreface, DaemonClientDeadline, DaemonHandshake, JsonRpcError, JsonRpcRequest, JsonRpcResponse, PROJECT_OPEN_RETRY_GRACE, PROJECT_OPEN_RETRY_INTERVAL, Result, - TraceDecayError, error_is_project_open_retryable, + TraceDecayError, error_is_project_open_retryable, tool_call_transport_error_is_retryable, }; /// Completed retryable problem results to observe before returning the typed @@ -467,8 +467,15 @@ pub async fn call_tool_within( .await } +/// Transport errors the one-shot client rides out on its own cadence: a +/// project open that has not finished (warming, deferred discovery, a +/// saturated open queue) and a retained project server retired mid-response +/// during a composition upgrade. The daemon types every one of these +/// `retryable: true`; a client that honours only the open subset reports the +/// upgrade window as a hard failure, which is what the reset-recovery journey +/// saw (`project_server_response_revoked` surfaced by `tracedecay tool`). fn is_project_open_retryable_error(error: &TraceDecayError) -> bool { - error_is_project_open_retryable(error) + error_is_project_open_retryable(error) || tool_call_transport_error_is_retryable(error) } /// Reconstruct a typed daemon tool refusal from the JSON-RPC error frame. @@ -725,6 +732,18 @@ mod tests { )) ); assert!(tool_call_transport_error_is_retryable(&revoked)); + assert!( + super::is_project_open_retryable_error(&revoked), + "the one-shot client rides out a mid-response retirement like a warming open" + ); + assert!( + super::project_open_retry_wait( + &Err(revoked), + tokio::time::Instant::now() + std::time::Duration::from_secs(5) + ) + .is_some(), + "a revoked response is re-sent, not returned" + ); } #[test] From cde4e57c26c12005ea8bd08264577efa032d389a Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 08:22:00 +0000 Subject: [PATCH 70/84] style(session-temporal): keep the test module last and rustfmt master Master run 35431539771 failed Check formatting (projector.rs, query.rs) and Clippy (items_after_test_module in query.rs) after #1844/#1845 merged without CI. Co-Authored-By: Claude Fable 5.1 --- .../projector.rs | 8 ++--- .../src/query.rs | 29 +++++++++---------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs index 10818733c1..632898ca24 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs @@ -119,11 +119,11 @@ impl SessionTemporalRefreshProjector for CanonicalSessionTemporalProjector { // Empty remaining range is a durable no-op: terminalize with an // empty complete progress batch instead of deferring forever. Ok(None) => canonical_noop_complete_effect(&recovery), - Err(error) if error.is_storage() => Err( - SessionTemporalRefreshProjectorError::retryable(format!( + Err(error) if error.is_storage() => { + Err(SessionTemporalRefreshProjectorError::retryable(format!( "source_busy: {error}" - )), - ), + ))) + } Err(_) => Err(SessionTemporalRefreshProjectorError::terminal( "projector_failed", )), diff --git a/crates/tracedecay-session-temporal-store/src/query.rs b/crates/tracedecay-session-temporal-store/src/query.rs index 367b14bb79..ae0ab6b4de 100644 --- a/crates/tracedecay-session-temporal-store/src/query.rs +++ b/crates/tracedecay-session-temporal-store/src/query.rs @@ -246,8 +246,7 @@ pub(super) async fn read_observations( // ceiling. Split until a single observation remains; that // observation is then a typed storage failure, not a retry // that looks like a busy source. - if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 - { + if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 { let mid = start + chunk.len() / 2; pending.push((mid, end)); pending.push((start, mid)); @@ -289,11 +288,18 @@ fn observation_prefetch_exceeded_materialization_limit(error: &SessionStoreError } } +/// The error `read_observation` raises for an id the store does not hold, reused +/// by callers that resolve prefetched observations out of a batch map. +pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { + storage_message( + PERSIST_OPERATION, + format!("source observation {} is missing", observation_id.as_str()), + ) +} + #[cfg(test)] mod tests { - use super::{ - PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage, - }; + use super::{PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage}; #[test] fn materialization_limit_is_the_prefetch_split_signal() { @@ -308,15 +314,8 @@ mod tests { assert!(observation_prefetch_exceeded_materialization_limit( &exceeded )); - assert!(!observation_prefetch_exceeded_materialization_limit(&locked)); + assert!(!observation_prefetch_exceeded_materialization_limit( + &locked + )); } } - -/// The error `read_observation` raises for an id the store does not hold, reused -/// by callers that resolve prefetched observations out of a batch map. -pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { - storage_message( - PERSIST_OPERATION, - format!("source observation {} is missing", observation_id.as_str()), - ) -} From 426ca8583f183de4fc6be60dc91eeca23a90f4c5 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 09:25:46 +0000 Subject: [PATCH 71/84] test(search-eval): re-pin the lexical baseline ranking receipts Batch run 35431957131 failed report_tests::baseline_report_retains_ raw_fallback_current_and_exact_ten_x_samples and the CLI compare journey: the query-fallback train/validation receipts drifted from the pins #1828 recorded on 3f71bccb2d. The drift is a ranking change, not a generation reseal: it reproduces on this branch and disappears with PR #1814's three extraction commits reverted (1ebadc81f9, 3daba38304, df9f190eea: bare receivers no longer invent same-file callers, module scope kept in self receiver types, extractor.rust.v11), so the lexical-graph workload's ordered rows legitimately moved with the extracted call graph. The fixed conceptual-miss set asserted by the same test is unchanged (train-015..033, validation-015..025). Co-Authored-By: Claude Fable 5.1 --- .../search_quality/query-lexical-graph-workload-v1.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json b/crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json index 1079dcf001..7870e2a69a 100644 --- a/crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json +++ b/crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json @@ -135,8 +135,8 @@ } ], "expected_query_fallback_digests": { - "train": "sha256:07c58382a8c5589236e2d29c91827825780dfc1aa11123bccd0f230805671e77", - "validation": "sha256:72b3500daa214348c143e25dab5f367c8bd723035e0471a48526bf3f55c9b54b" + "train": "sha256:f3744c56a8d35a13ddd9616eb949ae31b2bf745bf14b4d0bdc97c3e0ed139660", + "validation": "sha256:561eb59d2b58d5c935b28d0217dc18252a2d2c28e47f404cc59104e03c95625d" }, "profile_matrix": [ { From 7d678d8fba6f28508c71e5732d277d9f5542c82c Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 09:36:03 +0000 Subject: [PATCH 72/84] revert: "fix(codex): keep prepare Ready when the plugin CLI is absent" This reverts fdb756d61634f545b68686e181e9ca6ef6ac658d (#1812) and adds the unit guard that would have caught it. `NonInteractiveInstallOutcome::Ready` from `prepare_non_interactive_install` is a promise that Core apply can complete. For Codex, Core apply drives `codex plugin add` from `activate_deployed_host_registration`, so the promise is false whenever no `codex` binary resolves. #1812 returned `Ready` anyway, on the premise that a resolution failure belongs to activation's `HostCliUnavailable` rather than to a deferral. The lifecycle cannot honour that. `handle_install_command` calls `prepare_native_activation_if_needed` and, on `Ready`, opens the component transaction. Activation then fails with `HostCliUnavailable`, `rollback_component_set` runs, and (component_set.rs) it deliberately leaves the `RolledBack` journal and its registration backups on disk for the next command to reconcile. That backup pins `~/.codex/config.toml` and `~/.codex/plugins/cache//tracedecay//**` as they were *before* the operator acted. The error the CLI prints tells the operator to run `codex plugin add tracedecay@personal`, which mutates exactly those pinned paths. The next lifecycle command starts with `execute` -> `recover_host`, which replays the stale rollback; `restore_registration` finds each path matching neither the recorded original nor any recorded write intent and refuses with `StalePreview`. The remediation the failure itself prints is therefore impossible to follow, and every later install/update/repair for that host stays refused. Reproduced on ci/pr-batch-c with `codex` hidden from the host-program search path: `host_lifecycle_cli_acceptance::codex_stale_cache_remediation_executes_ on_the_current_stock_cli_and_converges_update` fails at the second install with `host bundle lifecycle failed: confirmed host lifecycle preview is stale or does not match apply (at .../host_component_registration.rs:1087)` -- the exact CI signature from run 35431957131. Deferring keeps the transaction closed until native activation is possible, which is why the journey passed on master. Activation's `HostCliUnavailable` is unchanged by this revert: `activate_deployed_host_registration` still calls `require_codex_plugin_cli`. `prepare_defers_when_no_plugin_cli_resolves` replaces #1812's deleted activation test with the inverse assertion at the boundary that actually regressed, so `--lib` catches this without the root-transport journey. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/agents/codex.rs | 33 ++++++-- .../src/agents/codex/tests.rs | 81 ++++++++++++------- 2 files changed, 81 insertions(+), 33 deletions(-) diff --git a/crates/tracedecay-agent-hosts/src/agents/codex.rs b/crates/tracedecay-agent-hosts/src/agents/codex.rs index 0599f13e06..d9f9a0246e 100644 --- a/crates/tracedecay-agent-hosts/src/agents/codex.rs +++ b/crates/tracedecay-agent-hosts/src/agents/codex.rs @@ -88,12 +88,35 @@ impl AgentIntegration for CodexIntegration { &self, ctx: &InstallContext, ) -> Result { - // Staging is ready for Core apply to drive `codex plugin add`. Whether - // that binary resolves is activation's `HostCliUnavailable`, not a - // deferral: collapsing every resolution failure into - // `DeferredUserAction` made prepare's outcome a property of PATH and - // stopped the lifecycle before the host CLI could run. install_codex_plugin(&ctx.home, &ctx.tracedecay_bin)?; + // Core apply drives `codex plugin add` when the host CLI is present. + // When it is not, stop with the same backtick remediation preflight + // uses so operators (and lifecycle tests) can activate natively. + // + // `Ready` here is a promise that Core apply can complete, so it must + // not be returned when no `codex` resolves. Returning it anyway opens + // a component transaction that can only die in activation with + // `HostCliUnavailable`; the rollback leaves a `RolledBack` journal + // whose registration backup pins `config.toml` and the versioned + // plugin cache as they were *before* the operator runs the printed + // `codex plugin add`. The next lifecycle command starts with + // `recover_host`, replays that stale rollback over the now-remediated + // host, and refuses with `StalePreview` -- making the remediation this + // very error prints impossible to follow. + if plugin_registry::require_codex_plugin_cli().is_err() { + let marketplace_name = codex_exact_personal_marketplace_name(&ctx.home) + .ok() + .flatten() + .unwrap_or_else(|| codex_cached_marketplace_name(&ctx.home)); + return Ok(NonInteractiveInstallOutcome::DeferredUserAction( + DeferredUserAction { + remediation: format!( + "Codex activates plugins through its native cache. Run `codex plugin add tracedecay@{marketplace_name}` after TraceDecay stages the source package." + ), + staged_paths: Vec::new(), + }, + )); + } Ok(NonInteractiveInstallOutcome::Ready) } diff --git a/crates/tracedecay-agent-hosts/src/agents/codex/tests.rs b/crates/tracedecay-agent-hosts/src/agents/codex/tests.rs index 7d2d42ffbc..bacb3d3374 100644 --- a/crates/tracedecay-agent-hosts/src/agents/codex/tests.rs +++ b/crates/tracedecay-agent-hosts/src/agents/codex/tests.rs @@ -979,23 +979,34 @@ fn codex_preflight_reports_inactive_cache_without_interactive_guidance() { assert!(CodexIntegration.interactive_removal_guidance().is_none()); } -/// Restrict host-program resolution to an empty directory. +/// Install an executable `codex` on the host-program search path only. /// -/// The directory has to outlive the guard. An ambient `codex` on the process -/// PATH must not be able to change the outcome under test. -fn hide_host_programs() -> ( - tempfile::TempDir, - tracedecay_runtime_core::config::HostProgramSearchPathGuard, -) { - let dir = tempfile::tempdir().unwrap(); - let guard = tracedecay_runtime_core::config::HostProgramSearchPathGuard::set(dir.path()); - (dir, guard) +/// Preparation is `Ready` exactly when Codex's own plugin CLI is present, so +/// the outcome under test is a property of the environment, not of the host +/// integration. CI runners carry no `codex` binary while a developer box +/// usually does; pin it here instead of reading whichever the machine has. +/// Only host program resolution sees this directory, the process `PATH` is +/// untouched. +fn install_fake_codex_cli( + dir: &Path, +) -> tracedecay_runtime_core::config::HostProgramSearchPathGuard { + let binary = dir.join(format!("codex{}", std::env::consts::EXE_SUFFIX)); + std::fs::write(&binary, "#!/bin/sh\nexit 0\n").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut permissions = std::fs::metadata(&binary).unwrap().permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(&binary, permissions).unwrap(); + } + tracedecay_runtime_core::config::HostProgramSearchPathGuard::set(dir) } #[test] fn prepare_stages_the_source_and_returns_ready_for_cli_activation() { let home = tempfile::tempdir().unwrap(); - let (_empty_path, _host_programs) = hide_host_programs(); + let cli_dir = tempfile::tempdir().unwrap(); + let _codex_cli = install_fake_codex_cli(cli_dir.path()); // Pre-existing user config: preparation runs before the component // transaction stages `config.toml`, so it must not write there, hook // trust is recorded by activation, inside the rollback boundary. @@ -1006,10 +1017,7 @@ fn prepare_stages_the_source_and_returns_ready_for_cli_activation() { let outcome = CodexIntegration .prepare_non_interactive_install(&install_ctx(home.path())) .unwrap(); - assert!( - matches!(outcome, NonInteractiveInstallOutcome::Ready), - "staging Codex must be Ready even when no host CLI resolves, got {outcome:?}" - ); + assert!(matches!(outcome, NonInteractiveInstallOutcome::Ready)); assert!(codex_plugin_manifest_path(home.path()).is_file()); assert!(codex_personal_marketplace_path(home.path()).is_file()); assert_eq!( @@ -1019,23 +1027,40 @@ fn prepare_stages_the_source_and_returns_ready_for_cli_activation() { ); } -/// A missing plugin CLI is an unavailable host, not a successful deferral. -/// Activation is the boundary that drives `codex plugin add`. +/// `Ready` promises Core apply can drive `codex plugin add`, so an +/// unresolvable plugin CLI must defer instead. +/// +/// Answering `Ready` opens a component transaction that can only die in +/// activation with `HostCliUnavailable`. Its rollback leaves a `RolledBack` +/// journal pinning `config.toml` and the versioned cache as they were before +/// the operator runs the remediation the failure prints, and the next +/// lifecycle command's `recover_host` then refuses the drifted host with +/// `StalePreview`. #[test] -fn activation_names_a_missing_plugin_cli_instead_of_deferring() { +fn prepare_defers_when_no_plugin_cli_resolves() { let home = tempfile::tempdir().unwrap(); - let (_empty_path, _host_programs) = hide_host_programs(); + // Resolution sees only this empty directory; the process `PATH` (which on + // a developer box usually does carry `codex`) is untouched. + let empty = tempfile::tempdir().unwrap(); + let _host_programs = + tracedecay_runtime_core::config::HostProgramSearchPathGuard::set(empty.path()); - let error = CodexIntegration - .activate_deployed_host_registration(&install_ctx(home.path())) - .expect_err("activation without a Codex CLI must fail"); - let TraceDecayError::HostCliUnavailable { program, lifecycle } = error else { - panic!( - "a missing Codex plugin CLI must stay HostCliUnavailable, not another error: {error}" - ); + let outcome = CodexIntegration + .prepare_non_interactive_install(&install_ctx(home.path())) + .unwrap(); + let NonInteractiveInstallOutcome::DeferredUserAction(deferred) = outcome else { + panic!("staging Codex without a resolvable plugin CLI must defer, got {outcome:?}"); }; - assert_eq!(program, "codex"); - assert_eq!(lifecycle, "codex plugin lifecycle"); + assert!( + deferred + .remediation + .contains("`codex plugin add tracedecay@personal`"), + "the deferral must print the executable remediation: {}", + deferred.remediation + ); + // The source is still staged: the operator's `codex plugin add` consumes it. + assert!(codex_plugin_manifest_path(home.path()).is_file()); + assert!(codex_personal_marketplace_path(home.path()).is_file()); } /// Activation must record hook trust even when Codex already reports the From 1196ebad4fe9772a756ffe6eeb76cd17c3164a9c Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 08:22:00 +0000 Subject: [PATCH 73/84] style(session-temporal): keep the test module last and rustfmt master Master run 35431539771 failed Check formatting (projector.rs, query.rs) and Clippy (items_after_test_module in query.rs) after #1844/#1845 merged without CI. Co-Authored-By: Claude Fable 5.1 --- .../projector.rs | 8 ++--- .../src/query.rs | 29 +++++++++---------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs index 10818733c1..632898ca24 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs @@ -119,11 +119,11 @@ impl SessionTemporalRefreshProjector for CanonicalSessionTemporalProjector { // Empty remaining range is a durable no-op: terminalize with an // empty complete progress batch instead of deferring forever. Ok(None) => canonical_noop_complete_effect(&recovery), - Err(error) if error.is_storage() => Err( - SessionTemporalRefreshProjectorError::retryable(format!( + Err(error) if error.is_storage() => { + Err(SessionTemporalRefreshProjectorError::retryable(format!( "source_busy: {error}" - )), - ), + ))) + } Err(_) => Err(SessionTemporalRefreshProjectorError::terminal( "projector_failed", )), diff --git a/crates/tracedecay-session-temporal-store/src/query.rs b/crates/tracedecay-session-temporal-store/src/query.rs index 367b14bb79..ae0ab6b4de 100644 --- a/crates/tracedecay-session-temporal-store/src/query.rs +++ b/crates/tracedecay-session-temporal-store/src/query.rs @@ -246,8 +246,7 @@ pub(super) async fn read_observations( // ceiling. Split until a single observation remains; that // observation is then a typed storage failure, not a retry // that looks like a busy source. - if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 - { + if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 { let mid = start + chunk.len() / 2; pending.push((mid, end)); pending.push((start, mid)); @@ -289,11 +288,18 @@ fn observation_prefetch_exceeded_materialization_limit(error: &SessionStoreError } } +/// The error `read_observation` raises for an id the store does not hold, reused +/// by callers that resolve prefetched observations out of a batch map. +pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { + storage_message( + PERSIST_OPERATION, + format!("source observation {} is missing", observation_id.as_str()), + ) +} + #[cfg(test)] mod tests { - use super::{ - PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage, - }; + use super::{PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage}; #[test] fn materialization_limit_is_the_prefetch_split_signal() { @@ -308,15 +314,8 @@ mod tests { assert!(observation_prefetch_exceeded_materialization_limit( &exceeded )); - assert!(!observation_prefetch_exceeded_materialization_limit(&locked)); + assert!(!observation_prefetch_exceeded_materialization_limit( + &locked + )); } } - -/// The error `read_observation` raises for an id the store does not hold, reused -/// by callers that resolve prefetched observations out of a batch map. -pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { - storage_message( - PERSIST_OPERATION, - format!("source observation {} is missing", observation_id.as_str()), - ) -} From c30cc2d6dcbea0d0ec5ad8ad13ee84d3f0905bb8 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 09:38:40 +0000 Subject: [PATCH 74/84] fix(mcp): drop the unreachable NotApplicable wire no-op The hook-runtime error boundary gained a branch that turned a `not_applicable` admission status into a successful JSON-RPC result, but no production path can produce that error. The only production constructor of `HostAdmissionStatus::NotApplicable` is `code_index_host_outcome`; its outcome reaches either the hook-event notification path (which returns no JSON-RPC response at all) or a replay worker pass, and neither converts an outcome into a `TraceDecayError`. The one hook-runtime call site that maps an outcome to an error, `hermes.rs`, is gated on `!commits_replay_record()`, and `NotApplicable` commits. Removing the branch and its synthetic test leaves the reachable half of the change: `classify_replay_pass` treating a closed `NotApplicable` that did not shrink the spool as `TerminalNoop`. Resolves the Codex P1 review finding on #1843 and corrects the CHANGELOG entry, which claimed a wire behaviour change that does not occur. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 6 +- crates/tracedecay-mcp/src/tool_errors.rs | 78 +----------------------- 2 files changed, 4 insertions(+), 80 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99bbe0890d..7c42d0b1c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1710,9 +1710,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- *(admission)* `NotApplicable` is a terminal no-op. Replay no longer backs - off on that closed status, and the MCP hook-runtime boundary returns a - successful no-op instead of a JSON-RPC failure. +- *(admission)* `NotApplicable` is a terminal no-op in the shared replay-pass + decision. A closed status that leaves the spool unchanged now stops until + the next kick instead of entering the retryable backoff arm. - *(code-index)* the background worker consults the typed publication-authority park instead of a loop-local bool, so a park it has not yet observed still diff --git a/crates/tracedecay-mcp/src/tool_errors.rs b/crates/tracedecay-mcp/src/tool_errors.rs index e0ab8cd06c..758792e860 100644 --- a/crates/tracedecay-mcp/src/tool_errors.rs +++ b/crates/tracedecay-mcp/src/tool_errors.rs @@ -44,28 +44,6 @@ fn value_has_semantic_error(value: &Value) -> bool { }) } -/// Successful MCP tool result for a closed `NotApplicable` admission. -/// -/// The status already means the demand does not apply. Publishing it as a -/// JSON-RPC error makes hosts retry a terminal no-op. -fn hook_runtime_not_applicable_noop(error: &TraceDecayError) -> Option { - let status = error.hook_runtime_status()?; - if HostAdmissionStatus::from_wire(status) != Some(HostAdmissionStatus::NotApplicable) { - return None; - } - let (reason_code, _, _) = error.hook_runtime_context()?; - let body = json!({ - "tool": "tracedecay_hook_runtime", - "status": status, - "reason_code": reason_code, - "retryable": false, - }); - Some(json!({ - "content": [{ "type": "text", "text": body.to_string() }], - "isError": false, - })) -} - /// Projects a hook-runtime error onto the structured JSON-RPC data object. /// /// The status is whatever the admission authority reported, carried through @@ -191,11 +169,6 @@ pub fn tool_error_response(id: Value, tool_name: &str, error: &TraceDecayError) Some(data), ); } - if tool_name == "tracedecay_hook_runtime" - && let Some(result) = hook_runtime_not_applicable_noop(error) - { - return JsonRpcResponse::success(id, result); - } if tool_name == "tracedecay_hook_runtime" && let Some(data) = structured_hook_error_data(error) { @@ -369,7 +342,7 @@ pub fn serialize_response_line(resp: &JsonRpcResponse) -> String { #[cfg(test)] mod tests { - use serde_json::{Value, json}; + use serde_json::json; use tracedecay_domain::errors::TraceDecayError; use super::tool_error_response; @@ -394,53 +367,4 @@ mod tests { "application_surface_invalid_request" ); } - - #[test] - fn not_applicable_hook_runtime_is_a_successful_noop_and_other_statuses_stay_errors() { - use tracedecay_sessions::admission::HostAdmissionStatus; - - let closed = TraceDecayError::hook_runtime_with_status( - "code_index_not_applicable", - true, - "projectless Hermes receipt host admission failed", - HostAdmissionStatus::NotApplicable.as_wire(), - ); - let wire = serde_json::to_value(tool_error_response( - json!(4), - "tracedecay_hook_runtime", - &closed, - )) - .expect("JSON-RPC wire response"); - - assert_eq!(wire["jsonrpc"], "2.0"); - assert_eq!(wire["id"], 4); - assert!(wire.get("error").is_none()); - assert_eq!(wire["result"]["isError"], false); - let text = wire["result"]["content"][0]["text"] - .as_str() - .expect("tool text"); - let body: Value = serde_json::from_str(text).expect("noop body"); - assert_eq!(body["tool"], "tracedecay_hook_runtime"); - assert_eq!(body["status"], "not_applicable"); - assert_eq!(body["reason_code"], "code_index_not_applicable"); - assert_eq!(body["retryable"], false); - assert!(body.get("error").is_none()); - assert!(!text.contains("failed")); - - let unavailable = TraceDecayError::hook_runtime_with_status( - "authority_unavailable", - true, - "host admission authority is unavailable", - HostAdmissionStatus::Unavailable.as_wire(), - ); - let failure = serde_json::to_value(tool_error_response( - json!(5), - "tracedecay_hook_runtime", - &unavailable, - )) - .expect("JSON-RPC wire response"); - assert!(failure.get("result").is_none()); - assert_eq!(failure["error"]["code"], -32603); - assert_eq!(failure["error"]["data"]["status"], "unavailable"); - } } From efdfdef0336b142dc3da1d9e78e12ba591cc9de2 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 09:47:35 +0000 Subject: [PATCH 75/84] fix(global-db): adopt a raw session only for projector-owned outputs adopt_owned_projection_raw_session defeats the raw upsert's `session_id = excluded.session_id` guard, so it must run only where the projector already owns the output. `existing.is_some()` was not that proof: classify_message_transition retains an equal pre-existing message row with no output state at all, and that path would have silently rewritten the session of a twin this projector never claimed instead of refusing typed. Gate on the output state's projector_owned flag, which is the same ownership bit the transition classifier uses. Co-Authored-By: Claude Fable 5.1 --- .../src/observation_projection/apply.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-global-db/src/observation_projection/apply.rs b/crates/tracedecay-global-db/src/observation_projection/apply.rs index 811ee1e92e..832db3d16a 100644 --- a/crates/tracedecay-global-db/src/observation_projection/apply.rs +++ b/crates/tracedecay-global-db/src/observation_projection/apply.rs @@ -1051,7 +1051,11 @@ async fn apply_rows( } }; if projected_message.provider != "hermes" && !preserve_protected_payload { - if existing.is_some() { + // Message-row presence is not projector ownership. An equal + // pre-existing row with no output state is retained without this + // projector ever having claimed the output, so its twin keeps the + // upsert's session guard and a disagreement stays a typed refusal. + if state.is_some_and(|state| state.projector_owned) { adopt_owned_projection_raw_session(conn, projected_message).await?; } upsert_projected_raw_message(conn, projected_message).await?; From 0cf720aca6833eaedd0578c9d8ed23596af0f909 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 09:47:46 +0000 Subject: [PATCH 76/84] fix(global-db): compare a raw twin's derived columns before trusting it owned_raw_twin_needs_rewrite reported a twin as current whenever its storage kind, session and content matched, so a row carrying a stale content_hash, snippet_text or index_text restored the audit checkpoint without ever running the canonical upsert. Hydration then kept failing that row with PayloadIntegrityMismatch and retrieval kept serving text the projector never wrote, indefinitely. Read those three columns in the batch and compare them against the same pure derivations upsert_inline_raw_message stores. Co-Authored-By: Claude Fable 5.1 --- .../src/observation_projection/state.rs | 15 ++++- .../src/schema_contract/invariants/audit.rs | 10 ++- .../invariants/released_rendering.rs | 62 +++++++++++++++++++ 3 files changed, 85 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation_projection/state.rs b/crates/tracedecay-global-db/src/observation_projection/state.rs index f3678fde55..e66e468fe3 100644 --- a/crates/tracedecay-global-db/src/observation_projection/state.rs +++ b/crates/tracedecay-global-db/src/observation_projection/state.rs @@ -796,6 +796,9 @@ pub(in super::super) struct ProjectionRawTwin { pub(in super::super) session_id: String, pub(in super::super) storage_kind: String, pub(in super::super) content: String, + pub(in super::super) content_hash: String, + pub(in super::super) snippet_text: String, + pub(in super::super) index_text: String, } pub(in super::super) struct ProjectionRowsBatch { @@ -899,7 +902,8 @@ pub(in super::super) async fn read_projection_rows_batch( let mut rows = conn .query( "SELECT raw.provider, raw.message_id, raw.session_id, raw.storage_kind, - COALESCE(raw.content, '') + COALESCE(raw.content, ''), raw.content_hash, raw.snippet_text, + raw.index_text FROM json_each(?1) AS requested CROSS JOIN lcm_raw_messages AS raw WHERE raw.provider = json_extract(requested.value, '$.provider') @@ -931,6 +935,15 @@ pub(in super::super) async fn read_projection_rows_batch( content: row .get(4) .map_err(|error| storage("decode projected raw twins", error))?, + content_hash: row + .get(5) + .map_err(|error| storage("decode projected raw twins", error))?, + snippet_text: row + .get(6) + .map_err(|error| storage("decode projected raw twins", error))?, + index_text: row + .get(7) + .map_err(|error| storage("decode projected raw twins", error))?, }, ); } diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs index d61bd06457..18ff885a23 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs @@ -932,9 +932,17 @@ fn owned_raw_twin_needs_rewrite( let Some(raw) = rows.raw_twin(&message.provider, &message.message_id) else { return Ok(true); }; + // The derived columns are pure functions of the same sanitized body, so a + // twin whose content matches can still carry a hash that fails hydration + // with `PayloadIntegrityMismatch` or retrieval text the projector never + // wrote. Compare what a fresh write stores, not content alone. Ok(raw.storage_kind != "inline" || raw.session_id != message.session_id - || raw.content != expected) + || raw.content != expected + || raw.content_hash != tracedecay_lcm::retrieval_content::projected_content_hash(&expected) + || raw.snippet_text + != tracedecay_lcm::retrieval_content::derived_text_for_snippet(&expected) + || raw.index_text != tracedecay_lcm::retrieval_content::derived_text_for_index(&expected)) } #[allow(clippy::too_many_arguments)] diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs index ee213c10fa..373c886329 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs @@ -712,6 +712,68 @@ mod tests { assert_eq!(stored_output(&snapshot, RECORD_ID).await, current); } + /// A twin whose content survived but whose derived columns did not still + /// fails hydration with `PayloadIntegrityMismatch`. Content equality alone + /// is not the twin a fresh projection write stores. + #[tokio::test] + async fn current_provenance_repairs_a_raw_twin_with_a_stale_hash() { + let directory = TempDir::new().unwrap(); + let runtime = HostAdmissionTestRuntimeV1::profile(directory.path()) + .await + .unwrap(); + seed(&runtime, &observation()).await.unwrap(); + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .expect("registered profile database"); + let snapshot = database.read_snapshot().await.unwrap(); + let current = stored_output(&snapshot, RECORD_ID).await; + drop(snapshot); + + let transaction = database + .runtime_database() + .begin_write_transaction("stale the raw twin derivations") + .await + .unwrap(); + let updated = transaction + .execute( + "UPDATE lcm_raw_messages + SET content_hash = 'stale', index_text = 'stale index' + WHERE provider = 'codex' AND message_id = ?1", + tracedecay_runtime_core::params![RECORD_ID], + ) + .await + .expect("stale the derived columns"); + assert_eq!(updated, 1); + transaction.commit().await.unwrap(); + + super::super::ensure_authority_invariants(database.runtime_database(), false, false) + .await + .expect("current provenance must repair a twin with stale derivations"); + + let snapshot = database.read_snapshot().await.unwrap(); + assert_eq!(stored_output(&snapshot, RECORD_ID).await, current); + let mut rows = snapshot + .query( + "SELECT content_hash, content FROM lcm_raw_messages + WHERE provider = 'codex' AND message_id = ?1", + tracedecay_runtime_core::params![RECORD_ID], + ) + .await + .expect("read the repaired twin"); + let row = rows + .next() + .await + .expect("read the repaired twin") + .expect("raw twin row"); + assert_eq!( + row.get::(0).unwrap(), + tracedecay_lcm::retrieval_content::projected_content_hash( + &row.get::(1).unwrap() + ), + "the repaired twin must carry the hash its content hydrates against" + ); + } + /// A drifted raw `session_id` is not a second owner of a provenance-bound /// output. The ingest upsert refuses that row; convergence still has to /// finish the rewrite or the stale message stays served. From ad95bff640770db6953db07c80f47d3d393345a4 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 13:25:13 +0000 Subject: [PATCH 77/84] fix(daemon): resend a warming project open as mounting The 500ms open bound leaves the open running and used to answer application.surface.unavailable. The typed CLI treats that as the answer, so cold storage_status and configuration writes fail the moment the bound elapses. Return the mounting refusal those clients already resend. Co-authored-by: Zack Jackson --- .../src/daemon/invocation_dispatch.rs | 91 ++++++++++++++++--- 1 file changed, 77 insertions(+), 14 deletions(-) diff --git a/crates/tracedecay/src/daemon/invocation_dispatch.rs b/crates/tracedecay/src/daemon/invocation_dispatch.rs index 61770b6a0f..517134a51c 100644 --- a/crates/tracedecay/src/daemon/invocation_dispatch.rs +++ b/crates/tracedecay/src/daemon/invocation_dispatch.rs @@ -118,9 +118,11 @@ fn lsp_project_open_wait_response( ) -> Option { match outcome { ProjectOpenWaitOutcome::Completed | ProjectOpenWaitOutcome::NotTracked => None, - ProjectOpenWaitOutcome::Failed(error) => Some(DaemonInvocationResponse::problem( + ProjectOpenWaitOutcome::Failed(error) => Some(project_open_refusal_response( request_id.to_owned(), - project_open_problem(&error, workflow_application, git_operation), + &error, + workflow_application, + git_operation, )), ProjectOpenWaitOutcome::Cancelled => Some(DaemonInvocationResponse::application_problem( request_id.to_owned(), @@ -228,9 +230,11 @@ async fn open_scope_set_cas_projects<'a>( Ok(Ok(_)) => {} Ok(Err(error)) => { record_project_open_refusal("multi_root_scope_set_compare_and_swap", &error); - return Err(DaemonInvocationResponse::problem( + return Err(project_open_refusal_response( request_id.to_owned(), - project_open_problem(&error, false, false), + &error, + false, + false, )); } Err(problem) => { @@ -272,9 +276,11 @@ async fn open_scope_set_cas_projects<'a>( Ok(Ok(project_server)) => servers.push(project_server), Ok(Err(error)) => { record_project_open_refusal("multi_root_scope_set_compare_and_swap", &error); - return Err(DaemonInvocationResponse::problem( + return Err(project_open_refusal_response( request_id.to_owned(), - project_open_problem(&error, false, false), + &error, + false, + false, )); } Err(problem) => { @@ -362,9 +368,11 @@ pub(super) async fn execute_portable_daemon_invocation( ); if let Err(error) = project_server { record_project_open_refusal(request.operation().as_str(), &error); - return DaemonInvocationResponse::problem( + return project_open_refusal_response( request_id, - project_open_problem(&error, workflow_application, git_operation), + &error, + workflow_application, + git_operation, ); } let project_route = project_route_for_handshake(handshake); @@ -423,9 +431,11 @@ pub(super) async fn execute_portable_daemon_invocation( } }; if let Err(error) = project_server { - return DaemonInvocationResponse::problem( + return project_open_refusal_response( request_id, - project_open_problem(&error, workflow_application, git_operation), + &error, + workflow_application, + git_operation, ); } let Ok((canonical_project_path, _)) = project_route_for_handshake(handshake) else { @@ -728,9 +738,11 @@ pub(super) async fn execute_daemon_invocation( ); if let Err(error) = project_server { record_project_open_refusal(request.operation().as_str(), &error); - return DaemonInvocationResponse::problem( + return project_open_refusal_response( request_id, - project_open_problem(&error, workflow_application, git_operation), + &error, + workflow_application, + git_operation, ); } let project_route = DaemonEngine::project_route(handshake); @@ -779,9 +791,11 @@ pub(super) async fn execute_daemon_invocation( } }; if let Err(error) = project_server { - return DaemonInvocationResponse::problem( + return project_open_refusal_response( request_id, - project_open_problem(&error, workflow_application, git_operation), + &error, + workflow_application, + git_operation, ); } let Ok((canonical_project_path, _)) = DaemonEngine::project_route(handshake) else { @@ -849,6 +863,37 @@ pub(super) async fn execute_daemon_invocation( .await } +/// A still-opening project is the mounting refusal the typed CLI re-sends +/// until its deadline. +/// +/// The 500 ms open bound answers "has this route published yet" and leaves +/// the open running. Mapping that miss to [`DaemonInvocationProblem::Unavailable`] +/// republishes `application.surface.unavailable`, which the typed client treats +/// as the answer, so a cold `storage_status` or configuration write fails the +/// moment the bound elapses. Terminal open failures stay on that problem. +fn project_open_refusal_response( + request_id: String, + error: &tracedecay_domain::errors::TraceDecayError, + workflow_application: bool, + git_operation: bool, +) -> DaemonInvocationResponse { + if error_is_project_open_retryable(error) { + return DaemonInvocationResponse::application_problem( + request_id, + tracedecay_contracts::ApplicationProblem::unavailable( + tracedecay_contracts::SafeDiagnostic { + code: tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE.to_owned(), + message: "The project runtime for this operation is still mounting".to_owned(), + }, + ), + ); + } + DaemonInvocationResponse::problem( + request_id, + project_open_problem(error, workflow_application, git_operation), + ) +} + fn project_open_problem( error: &tracedecay_domain::errors::TraceDecayError, workflow_application: bool, @@ -914,6 +959,24 @@ mod workflow_reset_tests { ); } + #[test] + fn warming_project_open_is_a_mounting_refusal_the_client_resends() { + let warming = project_warming_error(Path::new("/tmp/surface-fixture")); + let response = + project_open_refusal_response("request.warming".to_owned(), &warming, false, false); + let tracedecay_daemon_protocol::DaemonInvocationOutcome::ApplicationProblem { problem } = + response.outcome + else { + panic!("warming open must be an application problem, got {response:?}"); + }; + assert_eq!( + problem + .diagnostic() + .map(|diagnostic| diagnostic.code.as_str()), + Some(tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE) + ); + } + #[test] fn failed_project_open_keeps_the_terminal_problem_split() { let failed = tracedecay_domain::errors::TraceDecayError::Config { From 4a3f3b95fde0b6c29c2c6cd65ba00dd29b533260 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 13:25:14 +0000 Subject: [PATCH 78/84] test(code-index): assert directory pointer faults stay put The coalesced publication fault injects a directory, which rename cannot replace. The leftover truncated-file assertion read that directory and failed with EISDIR after the publication family had already held. Co-authored-by: Zack Jackson --- .../code_index_ignored_dependencies_test/flight_tests.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs index 8be7720786..28a66cd569 100644 --- a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs +++ b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs @@ -367,10 +367,9 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { .expect_err("follower publication fails closed"); assert_publication_error(owner_error); assert_publication_error(follower_error); - assert_eq!( - std::fs::read(&pointer_path).expect("faulted pointer remains"), - b"{", - "publication must not replace a pointer it did not observe" + assert!( + pointer_path.is_dir(), + "publication must not replace a directory pointer it did not observe" ); std::fs::remove_dir_all(&pointer_path).expect("remove faulted pointer node"); From e7c312474f1b3d9339cf1625fa8a90ed9ba63aa6 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 14:00:18 +0000 Subject: [PATCH 79/84] fix(global-db): page rebuild overlap reconciliation Rebuild activation read every staged session that already existed in one unbounded SELECT. The exact-SQL transport refuses a result set past MAX_QUERY_ROWS (10_000) or 64 MiB rather than truncating it, so a rebuild overlapping more history than that failed activation with InvalidOperation("exact SQL query materialization exceeded its limit") instead of reconciling. The predecessor query carried LIMIT 1; removing that predicate in 88139f6e60 removed the bound with it. Page by (provider, session_id) at REBUILD_PAGE_SIZE, the same bound the staging batches already use. Both the staged primary key and sessions are unique on that pair, so a page's writes never move a later page's cursor. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/observation_projection/rebuild.rs | 165 ++++++++++++++---- 1 file changed, 135 insertions(+), 30 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs index 8998fa4c7e..2501f2712d 100644 --- a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs +++ b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs @@ -2270,13 +2270,24 @@ async fn write_reconciled_sessions( /// A parallel SQL predicate used to report those conflicts as message /// `OutputCollision` values with `message_id = session:{id}`, which erased /// the field and sent session conflicts down the message-skip path. +/// Paged by `(provider, session_id)` because the exact-SQL transport refuses a +/// result set past `MAX_QUERY_ROWS` (10_000 rows) or 64 MiB, and a rebuild +/// overlapping more history than that would fail activation instead of +/// reconciling it. Both the staged primary key and `sessions` are unique on +/// that pair, so one page's writes never move a later page's cursor. async fn reconcile_overlapping_rebuild_sessions( conn: &impl Executor, generation: &str, ) -> ProjectionStoreResult<()> { - let mut overlaps = conn - .query( - "SELECT staged.session_json, + let mut cursor: Option<(String, String)> = None; + loop { + let (after_provider, after_session) = match cursor.as_ref() { + Some((provider, session_id)) => (Some(provider.as_str()), Some(session_id.as_str())), + None => (None, None), + }; + let mut overlaps = conn + .query( + "SELECT staged.session_json, active.provider, active.session_id, active.project_key, active.project_path, active.title, active.started_at, active.ended_at, active.transcript_path, active.metadata_json, @@ -2285,38 +2296,58 @@ async fn reconcile_overlapping_rebuild_sessions( FROM observation_projection_rebuild_sessions AS staged JOIN sessions AS active ON active.provider = staged.provider AND active.session_id = staged.session_id - WHERE staged.projector_version = ?1 AND staged.generation = ?2", - params![SESSION_MESSAGE_PROJECTOR_VERSION, generation], - ) - .await - .map_err(|error| storage("read overlapping projection sessions", error))?; - let mut updates = Vec::new(); - while let Some(row) = overlaps - .next() - .await - .map_err(|error| storage("read overlapping projection sessions", error))? - { - let staged_json: String = row - .get(0) + WHERE staged.projector_version = ?1 AND staged.generation = ?2 + AND (?3 IS NULL + OR staged.provider > ?3 + OR (staged.provider = ?3 AND staged.session_id > ?4)) + ORDER BY staged.provider, staged.session_id + LIMIT ?5", + params![ + SESSION_MESSAGE_PROJECTOR_VERSION, + generation, + after_provider, + after_session, + REBUILD_PAGE_SIZE + ], + ) + .await .map_err(|error| storage("read overlapping projection sessions", error))?; - let staged: SessionRecord = decode_json(&staged_json, "decode staged projection session")?; - let actual = decode_overlapping_session(&row)?; - let expected = canonicalize_session_project_paths(&staged); - let normalized_actual = canonicalize_session_project_paths(&actual); - let merged = - reconcile_session_rows_detailed(&normalized_actual, &expected).map_err(|conflict| { - ProjectionStoreError::SessionOutputCollision { + let mut updates = Vec::new(); + let mut scanned = 0_i64; + let mut last = None; + while let Some(row) = overlaps + .next() + .await + .map_err(|error| storage("read overlapping projection sessions", error))? + { + let staged_json: String = row + .get(0) + .map_err(|error| storage("read overlapping projection sessions", error))?; + let staged: SessionRecord = + decode_json(&staged_json, "decode staged projection session")?; + let actual = decode_overlapping_session(&row)?; + scanned += 1; + last = Some((actual.provider.clone(), actual.session_id.clone())); + let expected = canonicalize_session_project_paths(&staged); + let normalized_actual = canonicalize_session_project_paths(&actual); + let merged = reconcile_session_rows_detailed(&normalized_actual, &expected).map_err( + |conflict| ProjectionStoreError::SessionOutputCollision { provider: expected.provider.clone(), session_id: expected.session_id.clone(), field: conflict.field(), - } - })?; - if merged != actual { - updates.push(merged); + }, + )?; + if merged != actual { + updates.push(merged); + } } + drop(overlaps); + write_reconciled_sessions(conn, &updates).await?; + if scanned < REBUILD_PAGE_SIZE { + return Ok(()); + } + cursor = last; } - drop(overlaps); - write_reconciled_sessions(conn, &updates).await } async fn activate_rebuild_sessions( @@ -2641,12 +2672,19 @@ async fn activate_rebuild_dispositions( #[cfg(test)] #[allow(clippy::unwrap_used)] mod activation_tests { - use super::{SESSION_MESSAGE_PROJECTOR_VERSION, activate_rebuild_sessions}; + use super::{ + REBUILD_PAGE_SIZE, SESSION_MESSAGE_PROJECTOR_VERSION, activate_rebuild_sessions, + reconcile_overlapping_rebuild_sessions, + }; use crate::tests::harness::RegisteredGlobalDbHarness; use tracedecay_runtime_core::db::engine::{Executor, params}; use tracedecay_store::{ProjectionStoreError, SessionRecord}; const SESSION_ID: &str = "002bd803-dc62-46e2-b66a-a61cc282f0dc"; + /// One past the exact-SQL transport's `MAX_QUERY_ROWS` + /// (`tracedecay-rusqlite-runtime/src/exact_sql/mod.rs`), which refuses a + /// result set rather than truncating it. + const OVERLAPS_PAST_TRANSPORT_ROW_CAP: i64 = 10_001; fn session(transcript_path: Option<&str>, title: Option<&str>) -> SessionRecord { SessionRecord { @@ -2693,6 +2731,73 @@ mod activation_tests { .unwrap(); } + #[tokio::test] + async fn overlap_reconciliation_pages_past_the_transport_row_cap() { + const GENERATION: &str = "generation.session-overlap-paging"; + let harness = RegisteredGlobalDbHarness::open("session-overlap-paging").await; + let mut staged_rows = Vec::with_capacity(OVERLAPS_PAST_TRANSPORT_ROW_CAP as usize); + for index in 0..OVERLAPS_PAST_TRANSPORT_ROW_CAP { + let session_id = format!("session.{index:05}"); + let active = SessionRecord { + session_id: session_id.clone(), + ..session(None, None) + }; + assert!(harness.registered.upsert_session(&active).await); + staged_rows.push(SessionRecord { + session_id, + title: Some(format!("composer {index:05}")), + ..session(None, None) + }); + } + let transaction = harness.registered.begin_write_transaction().await.unwrap(); + transaction + .execute( + "INSERT INTO observation_projection_rebuilds ( + projector_version, generation, frontier_sequence, state + ) VALUES (?1, ?2, 0, 'ready')", + params![SESSION_MESSAGE_PROJECTOR_VERSION, GENERATION], + ) + .await + .unwrap(); + let staged_json = serde_json::to_string(&staged_rows).unwrap(); + transaction + .execute( + "INSERT INTO observation_projection_rebuild_sessions ( + projector_version, generation, provider, session_id, session_json + ) + SELECT ?1, ?2, + json_extract(staged.value, '$.provider'), + json_extract(staged.value, '$.session_id'), + staged.value + FROM json_each(?3) AS staged", + params![ + SESSION_MESSAGE_PROJECTOR_VERSION, + GENERATION, + staged_json.as_str() + ], + ) + .await + .unwrap(); + + reconcile_overlapping_rebuild_sessions(&transaction, GENERATION) + .await + .expect("an overlap larger than one transport page must still reconcile"); + + let mut rows = transaction + .query( + "SELECT COUNT(*) FROM sessions + WHERE provider = 'cursor' AND title LIKE 'composer %'", + (), + ) + .await + .unwrap(); + let reconciled = rows.next().await.unwrap().unwrap().get::(0).unwrap(); + assert_eq!( + reconciled, OVERLAPS_PAST_TRANSPORT_ROW_CAP, + "every overlapping session must be reconciled, not one page of {REBUILD_PAGE_SIZE}", + ); + } + #[tokio::test] async fn activation_names_the_session_field_instead_of_a_message_collision() { let harness = RegisteredGlobalDbHarness::open("session-collision-field").await; From 17b102208151832ec2a6ad142d317c9f7146483b Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 13:57:58 +0000 Subject: [PATCH 80/84] fix(sessions): drop the u64 identity conversion clippy rejects CI clippy (rust 1.97, -D warnings) fails master since #1857: useless_conversion on the JSONL partial-window test's file length, which is already u64. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit cb730a980cf66f3334bab4df729c58b101d0e6ae) --- .../runtime/observation/jsonl_observation_admission/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tracedecay-sessions/src/runtime/observation/jsonl_observation_admission/tests.rs b/crates/tracedecay-sessions/src/runtime/observation/jsonl_observation_admission/tests.rs index b5a72dfff6..6c33688210 100644 --- a/crates/tracedecay-sessions/src/runtime/observation/jsonl_observation_admission/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/observation/jsonl_observation_admission/tests.rs @@ -1026,7 +1026,7 @@ async fn cursor_cas_lost_on_a_partially_covered_window_replays_the_tail() { }) ) .unwrap(); - let len = u64::try_from(std::fs::metadata(&path).unwrap().len()).unwrap(); + let len = std::fs::metadata(&path).unwrap().len(); let spy = SeamSpyAdmission::default(); spy.script_peer_covers_batch_prefix(); From 309059930caff9f4c24aff51c3642aa23e94af5d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 14:50:32 +0000 Subject: [PATCH 81/84] test(mcp): pin describe previews and clone publish wait Session describe now reports the bounded snippet and stored length, not a blank stub. A generation bump can answer search_failed while clone publication is still retiring; wait for the stale cursor. Co-authored-by: Zack Jackson --- .../mcp_handler_test/graph_query_test.rs | 77 ++++++++++++++----- .../mcp_handler_test/lcm_describe_behavior.rs | 34 +++++--- 2 files changed, 79 insertions(+), 32 deletions(-) diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/graph_query_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/graph_query_test.rs index d5bb7ea230..defff94f3c 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/graph_query_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/graph_query_test.rs @@ -104,6 +104,12 @@ pub struct PlainValue; .await } +fn clone_family_lane_still_publishing(error: &str) -> bool { + error.contains("search_failed") + || error.contains("generation_unverified") + || error.contains("code_index_unavailable") +} + async fn shutdown_graph_fixture(fixture: GraphQueryFixture) { fixture.production.harness.shutdown().await; } @@ -1577,28 +1583,57 @@ async fn redundancy_pull_request_scope_shares_one_budget_and_resumes_changed_fam .server(fixture.project_root()) .expect("production graph-query server"); warm_code_index_search(&server, "generation_bump").await; - let stale = call_production_tool( - &fixture, - "tracedecay_redundancy", - json!({ - "project_id": project_id, - "repository_id": repository_id, - "match_classes": ["conservative_exact"], - "scope": scope, - "include_generated_paths": true, - "family_limit": 10, - "member_limit": 10, - "work_limit": 4, - "cursor": stale_cursor, - }), - None, - None, - ) - .await - .expect_err("a prior-generation pull-request cursor must be stale"); + let stale_args = json!({ + "project_id": project_id, + "repository_id": repository_id, + "match_classes": ["conservative_exact"], + "scope": scope, + "include_generated_paths": true, + "family_limit": 10, + "member_limit": 10, + "work_limit": 4, + "cursor": stale_cursor, + }); + // Search lane coverage can seal while clone-family publication is still + // retiring the previous generation. That window answers `search_failed` + // or `generation_unverified`; the stale cursor's terminal is + // `generation_unavailable` once the successor artifact can be read. + let mut last = String::new(); + for _ in 0..40 { + match call_production_tool( + &fixture, + "tracedecay_redundancy", + stale_args.clone(), + None, + None, + ) + .await + { + Err(error) => { + let rendered = error.to_string(); + if rendered.contains("generation_unavailable") { + last = rendered; + break; + } + if clone_family_lane_still_publishing(&rendered) { + last = rendered; + tokio::time::sleep(Duration::from_millis(250)).await; + continue; + } + panic!("stale pull-request cursor failed closed: {rendered}"); + } + Ok(value) => { + last = format!( + "successor still served the prior cursor: {}", + extract_text(&value.value) + ); + tokio::time::sleep(Duration::from_millis(250)).await; + } + } + } assert!( - stale.to_string().contains("generation_unavailable"), - "{stale}" + last.contains("generation_unavailable"), + "prior-generation pull-request cursor must be stale, last={last}" ); shutdown_graph_fixture(fixture).await; diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_describe_behavior.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_describe_behavior.rs index 04b77576fd..975acae89f 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_describe_behavior.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_describe_behavior.rs @@ -246,7 +246,19 @@ async fn describe_raw(server: &Arc, arguments: Value) -> Value { handle_real_server_tool_call_raw(server, "tracedecay_lcm_describe", arguments).await } +fn externalized_tool_preview(payload_ref: &str) -> String { + let body = format!("{SECRET} {}", "payload ".repeat(40_000)); + format!( + "[Externalized LCM ingest payload: kind=tool_result; field=content; chars={}; bytes={}; ref={payload_ref}]", + body.chars().count(), + body.len() + ) +} + fn session_document(node_id: &str, payload_ref: &str) -> Value { + let tool_preview = externalized_tool_preview(payload_ref); + let tool_preview_chars = tool_preview.chars().count(); + let source_chars = SOURCE_BODY.chars().count(); json!({ "description": { "external_payload": null, @@ -257,13 +269,13 @@ fn session_document(node_id: &str, payload_ref: &str) -> Value { "raw_message_count": 2, "raw_messages": [ { - "content_preview": "", + "content_preview": SOURCE_BODY, "content_range": { - "limit": 0, + "limit": source_chars, "offset": 0, - "returned_chars": 0, - "total_chars": SOURCE_BODY.len(), - "truncated": true + "returned_chars": source_chars, + "total_chars": source_chars, + "truncated": false }, "message_id": SOURCE_ID, "payload_ref": null, @@ -272,12 +284,12 @@ fn session_document(node_id: &str, payload_ref: &str) -> Value { "store_id": 1 }, { - "content_preview": "", + "content_preview": tool_preview, "content_range": { - "limit": 0, + "limit": tool_preview_chars, "offset": 0, - "returned_chars": 0, - "total_chars": 180, + "returned_chars": tool_preview_chars, + "total_chars": 320_040, "truncated": true }, "message_id": TOOL_ID, @@ -298,7 +310,7 @@ fn session_document(node_id: &str, payload_ref: &str) -> Value { "depth": 0, "node_id": node_id, "source_count": 1, - "summary_preview": "" + "summary_preview": SUMMARY } ], "target": "session" @@ -402,7 +414,7 @@ fn external_payload_document(payload_ref: &str, content_hash: &str) -> Value { "byte_count": 320_040, "char_count": 320_040, "content_hash": content_hash, - "content_preview": "", + "content_preview": externalized_tool_preview(payload_ref), "created_at": "", "kind": "tool_result", "message_id": TOOL_ID, From 44eacdb6e4f10f24591c2f75cf199163666d7afe Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 14:53:13 +0000 Subject: [PATCH 82/84] test(mcp): expect the bounded describe preview #1817 restores The #1722 describe proof pinned the blank preview that the registered renderer regressed to. #1817 restores the stored bounded snippet, so the session overview and the external-payload target now show the snippet (the externalized-payload placeholder, never the payload body). Co-Authored-By: Claude Fable 5.1 --- .../mcp_handler_test/lcm_describe_behavior.rs | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_describe_behavior.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_describe_behavior.rs index 04b77576fd..324bc17c0e 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_describe_behavior.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_describe_behavior.rs @@ -247,6 +247,9 @@ async fn describe_raw(server: &Arc, arguments: Value) -> Value { } fn session_document(node_id: &str, payload_ref: &str) -> Value { + let external_placeholder = format!( + "[Externalized LCM ingest payload: kind=tool_result; field=content; chars=320040; bytes=320040; ref={payload_ref}]" + ); json!({ "description": { "external_payload": null, @@ -257,13 +260,13 @@ fn session_document(node_id: &str, payload_ref: &str) -> Value { "raw_message_count": 2, "raw_messages": [ { - "content_preview": "", + "content_preview": SOURCE_BODY, "content_range": { - "limit": 0, + "limit": SOURCE_BODY.len(), "offset": 0, - "returned_chars": 0, + "returned_chars": SOURCE_BODY.len(), "total_chars": SOURCE_BODY.len(), - "truncated": true + "truncated": false }, "message_id": SOURCE_ID, "payload_ref": null, @@ -272,12 +275,12 @@ fn session_document(node_id: &str, payload_ref: &str) -> Value { "store_id": 1 }, { - "content_preview": "", + "content_preview": external_placeholder, "content_range": { - "limit": 0, + "limit": external_placeholder.len(), "offset": 0, - "returned_chars": 0, - "total_chars": 180, + "returned_chars": external_placeholder.len(), + "total_chars": 320_040, "truncated": true }, "message_id": TOOL_ID, @@ -298,7 +301,7 @@ fn session_document(node_id: &str, payload_ref: &str) -> Value { "depth": 0, "node_id": node_id, "source_count": 1, - "summary_preview": "" + "summary_preview": SUMMARY } ], "target": "session" @@ -402,7 +405,9 @@ fn external_payload_document(payload_ref: &str, content_hash: &str) -> Value { "byte_count": 320_040, "char_count": 320_040, "content_hash": content_hash, - "content_preview": "", + "content_preview": format!( + "[Externalized LCM ingest payload: kind=tool_result; field=content; chars=320040; bytes=320040; ref={payload_ref}]" + ), "created_at": "", "kind": "tool_result", "message_id": TOOL_ID, From 024364a1f112f0d84970e0444d9895efcc36bb10 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 15:35:32 +0000 Subject: [PATCH 83/84] fix(code-index): let a retired clone staging discard converge `discard_incompatible_staging` read `symlink_metadata` and reported an unavailable authority when the staging file was already gone. A concurrent clone-successor build retires that path, so the discard found its own goal state and failed on it. The caller in `begin_clone_successor` uses the discard to recover an incompatible staging database and reopen it; the refusal aborted that recovery instead, and the clone-family lane answered `search_failed` with `retryable=false`. Treat an absent staging path as the end state and sweep its sidecars, the way `prepare_absent_text_artifact_staging` and the sidecar sweep already read `NotFound`. The clone-similarity warmup reports `Pending` when the advance budget runs out, so a caller learns the projection is unfinished and may retry. An `AuthorityUnavailable` advance failure is that same unfinished state, but the executor collapsed it into `Internal`, telling every caller never to retry a window one background pass resolves. Both clone lanes now keep `Pending`'s `generation_unverified` verdict for it. Evidence: with the redundancy tests pinned to two CPUs under load, `discard_incompatible_staging` returned `AuthorityUnavailable("No such file or directory (os error 2)")` through `begin_clone_successor` -> `advance_artifact_text_serving` -> `finish_clone_similarity_warmup_for_request`, and the tool answered `reason_code=search_failed retryable=false`. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/code_index_executor.rs | 20 +++++++++++++ .../src/code_index_scheduler/serving.rs | 30 ++++++++++++++----- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_executor.rs b/crates/tracedecay-code-index-runtime/src/code_index_executor.rs index 0438ee6c2e..a9e79db843 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_executor.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_executor.rs @@ -1501,6 +1501,16 @@ where code_search::CodeIndexSearchUnavailableReasonV1::GenerationUnverified, ); } + // A contended or already-retired staging artifact leaves this + // generation's clone projection unfinished. That is the same + // state `Pending` reports above, so it keeps `Pending`'s + // retryable verdict; `Internal` told callers never to retry a + // window that resolves itself within one background pass. + Err(RetrievalPortError::AuthorityUnavailable(_)) => { + return unavailable( + code_search::CodeIndexSearchUnavailableReasonV1::GenerationUnverified, + ); + } Err(_) => { return unavailable(code_search::CodeIndexSearchUnavailableReasonV1::Internal); } @@ -1697,6 +1707,16 @@ where code_search::CodeIndexSearchUnavailableReasonV1::GenerationUnverified, ); } + // A contended or already-retired staging artifact leaves this + // generation's clone projection unfinished. That is the same + // state `Pending` reports above, so it keeps `Pending`'s + // retryable verdict; `Internal` told callers never to retry a + // window that resolves itself within one background pass. + Err(RetrievalPortError::AuthorityUnavailable(_)) => { + return unavailable( + code_search::CodeIndexSearchUnavailableReasonV1::GenerationUnverified, + ); + } Err(_) => { return unavailable(code_search::CodeIndexSearchUnavailableReasonV1::Internal); } diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs index bf72387e38..cb082cfd60 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs @@ -1312,15 +1312,29 @@ impl DaemonCodeTextArtifactStoreV1 { } let _lock = self.acquire_store_write_lock()?; checkpoint_text_artifact_control(control)?; - let metadata = staging_path - .symlink_metadata() - .map_err(text_artifact_unavailable)?; - if !metadata.file_type().is_file() { - return Err(RetrievalPortError::Contract( - "incompatible text-artifact staging path is not a regular file".to_owned(), - )); + match staging_path.symlink_metadata() { + Ok(metadata) if metadata.file_type().is_file() => { + retire_text_artifact_staging_family(staging_path) + .map_err(text_artifact_unavailable)?; + } + Ok(_) => { + return Err(RetrievalPortError::Contract( + "incompatible text-artifact staging path is not a regular file".to_owned(), + )); + } + // A concurrent build may retire this staging file first. Discard + // wants it gone, so finding it already gone is the end state, not + // an unavailable authority: reporting one aborts the caller's + // reopen and the clone lane answers a non-retryable failure for a + // state that has already resolved. Sidecars can outlive the + // database after a crash, so sweep them the way + // `prepare_absent_text_artifact_staging` does. + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + clear_text_artifact_staging_sidecars(staging_path) + .map_err(text_artifact_unavailable)?; + } + Err(error) => return Err(text_artifact_unavailable(error)), } - retire_text_artifact_staging_family(staging_path).map_err(text_artifact_unavailable)?; DaemonCodeIndexPublicationStoreV1::sync_directory(&artifacts_root) .map_err(text_artifact_unavailable) } From f0a144a2f50551f4439cb50ae079219ef7052302 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 15:38:16 +0000 Subject: [PATCH 84/84] test(mcp): wait only on the retryable clone-lane verdict search_failed and code_index_unavailable are retryable=false; treating them as a publication window would hide a real failure now that the clone lanes answer generation_unverified while the projection finishes. Co-Authored-By: Claude Fable 5.1 --- .../tests/mcp_suite/mcp_handler_test/graph_query_test.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/graph_query_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/graph_query_test.rs index defff94f3c..2ea3be05ea 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/graph_query_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/graph_query_test.rs @@ -105,9 +105,7 @@ pub struct PlainValue; } fn clone_family_lane_still_publishing(error: &str) -> bool { - error.contains("search_failed") - || error.contains("generation_unverified") - || error.contains("code_index_unavailable") + error.contains("generation_unverified") } async fn shutdown_graph_fixture(fixture: GraphQueryFixture) {