From 8981020db6ee5e0518cd360dca644a745bbb3899 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 03:46:37 +0000 Subject: [PATCH 1/4] test(ci): close three master-tip readiness races HTTP reset checks ride out a still-warming project open instead of treating the first after_delay answer as the terminal. Ignored-dependency publication faults a pointer a background pass cannot rename over. Killing a test daemon now signals its process group, and the next spawn waits out a socket that is still closing. Co-authored-by: Zack Jackson --- crates/tracedecay/tests/common/mod.rs | 18 +++ .../flight_tests.rs | 33 ++++- .../transport_boundaries.rs | 133 +++++++++++++----- 3 files changed, 143 insertions(+), 41 deletions(-) diff --git a/crates/tracedecay/tests/common/mod.rs b/crates/tracedecay/tests/common/mod.rs index 28bbc596c4..201a7a2b38 100644 --- a/crates/tracedecay/tests/common/mod.rs +++ b/crates/tracedecay/tests/common/mod.rs @@ -783,11 +783,29 @@ impl Drop for TestChildProcess { } /// PID-directed stop: survives `process_group(0)` / `setsid` detachment. +/// +/// The child is the leader of its own group. Killing only that pid leaves +/// helper children that still hold the listen socket, so the next spawn +/// observes a connectable daemon after this process has already been reaped. fn terminate_and_reap(child: &mut Child) -> std::io::Result { if let Ok(Some(status)) = child.try_wait() { return Ok(status); } + #[cfg(unix)] + { + let pid = child.id(); + if pid != 0 { + // SAFETY: `pid` is this live child. Negating it targets the + // process group `process_group(0)` created with that pid as + // leader. ESRCH is ignored: setpgid may not have run yet, and the + // pid kill below still stops the leader. + unsafe { + libc::kill(-(pid as i32), libc::SIGKILL); + } + } + } + if let Err(kill_err) = child.kill() { if let Some(status) = child.try_wait()? { return Ok(status); diff --git a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs index bc1388df7f..77309c5c22 100644 --- a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs +++ b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs @@ -2,6 +2,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Condvar, Mutex}; use tracedecay_code_index::production::CodeIndexProductionErrorV1; +use tracedecay_code_index_retention::code_index_generations::try_acquire_code_generation_store_lock; use super::*; @@ -157,6 +158,33 @@ fn assert_publication_error(error: CodeIndexSchedulerErrorV1) { ); } +/// Hold the only background permit once no pass is in flight. +/// +/// Text seating keeps `reconcile_in_progress` after it drops the scheduler +/// mutex, and that pass can still rename a valid active pointer. A truncated +/// pointer written in that window is not a closed fault. Occupying the permit +/// while the owner has not entered its pass stops that rewrite. +async fn hold_idle_background_admission( + registry: &CodeIndexSchedulerRegistryV1, +) -> tokio::sync::OwnedSemaphorePermit { + let admission = registry.background_reconcile_admission(); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + if registry.memory_stats().await.reconciling_worktrees == 0 + && let Ok(permit) = admission.clone().try_acquire_owned() + { + if registry.memory_stats().await.reconciling_worktrees == 0 { + return permit; + } + } + assert!( + std::time::Instant::now() <= deadline, + "background reconcile did not go idle before publication fault injection" + ); + tokio::time::sleep(Duration::from_millis(2)).await; + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn aborted_flight_owner_wakes_follower_and_allows_a_fresh_owner() { let fixture = fixture(); @@ -242,6 +270,8 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { let registry = Arc::new(mount(fixture.path(), &store, 1).await); let baseline = latest(®istry, fixture.path()).await; let request = request_for(&baseline, "pkg"); + let idle_admission = hold_idle_background_admission(®istry).await; + registry.clear_pending_wake_for_scope(&request.scope).await; let hold = SchedulerHold::acquire(®istry, fixture.path()).await; let (owner_control, owner_entered) = BlockingNthControl::new(4); @@ -297,8 +327,6 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { // and any writer that starts after it is released reads the corruption // under the lock and refuses instead of overwriting it. let pointer_bytes = { - use tracedecay_code_index_retention::code_index_generations::try_acquire_code_generation_store_lock; - let store_lock = tokio::time::timeout(Duration::from_secs(5), async { loop { if let Some(lock) = try_acquire_code_generation_store_lock(&scoped_store) @@ -316,6 +344,7 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { drop(store_lock); pointer_bytes }; + drop(idle_admission); owner_control.release(); hold.release(); diff --git a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/transport_boundaries.rs b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/transport_boundaries.rs index daca6a9303..dabdfd4a42 100644 --- a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/transport_boundaries.rs +++ b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/transport_boundaries.rs @@ -377,6 +377,62 @@ fn problem_envelope(payload: &Value, context: &str) -> Value { panic!("{context}: no typed problem envelope in the payload: {payload}") } +/// True when the daemon has not published the project open yet. +/// +/// The open wait on a connection is 500 ms. Past that, the surface answers +/// `unavailable` / `after_delay` and leaves the open running. The CLI rides +/// that refusal out; a raw HTTP or SDK call does not. A restart's first packet +/// can therefore be warming even when the store's recorded terminal is +/// `reset_required`. +fn retryable_pre_admission_unavailable(payload: &Value) -> bool { + let problem = &payload["problem"]; + problem["kind"] == "unavailable" + && problem["retry"] == "after_delay" + && problem["terminality"] == "pre_admission" +} + +/// Polls until the open publishes a non-retryable problem, then returns it. +/// +/// Bounded by the same 15 s a CLI tool call gives a cold open. A refusal that +/// stays retryable past that bound is a real failure, not a slow open. +fn await_settled_problem(context: &str, mut fetch: impl FnMut() -> Value) -> Value { + let deadline = Instant::now() + Duration::from_secs(15); + loop { + let payload = fetch(); + if !retryable_pre_admission_unavailable(&payload) { + return payload; + } + assert!( + Instant::now() < deadline, + "{context}: project open stayed retryable unavailable past the open grace: {payload}" + ); + std::thread::sleep(Duration::from_millis(50)); + } +} + +fn await_sdk_reset_problem( + context: &str, + client: &Client, + request: &::Request, +) -> (String, Value) { + let deadline = Instant::now() + Duration::from_secs(15); + loop { + let error = client + .execute::(request) + .expect_err("a refused store must not read as a healthy status"); + let (kind, envelope) = sdk_problem(error, context); + let payload = problem_envelope(&envelope, context); + if !retryable_pre_admission_unavailable(&payload) { + return (kind, envelope); + } + assert!( + Instant::now() < deadline, + "{context}: project open stayed retryable unavailable past the open grace: {payload}" + ); + std::thread::sleep(Duration::from_millis(50)); + } +} + /// Arms the daemon's one-shot fact-commit barrier, runs `request` on its own /// thread, holds the committed effect there until the request's own deadline /// has certainly expired, then releases it and returns what `request` produced. @@ -665,25 +721,23 @@ fn reset_required_survives_http_mcp_and_rust_sdk_across_restart() { "a typed HTTP terminal must not be reported as success: status {http_status}, body {http_body}" ); - let mcp_response = mcp_tool_call( - &home_path, - &project_path, - "tracedecay_storage_status", - &storage_status_body, - None, - ); - super::assert_reset_required( - &problem_envelope(&mcp_payload(&mcp_response), "MCP reset required"), - "MCP stdio host, first observation", - ); + let mcp_problem = await_settled_problem("MCP stdio host, first observation", || { + let mcp_response = mcp_tool_call( + &home_path, + &project_path, + "tracedecay_storage_status", + &storage_status_body, + None, + ); + problem_envelope(&mcp_payload(&mcp_response), "MCP reset required") + }); + super::assert_reset_required(&mcp_problem, "MCP stdio host, first observation"); let client = sdk_client(&mount, &identity); let request = serde_json::from_value(storage_status_body.clone()).expect("canonical storage status"); - let sdk_error = client - .execute::(&request) - .expect_err("a refused store must not read as a healthy status"); - let (sdk_kind, sdk_envelope) = sdk_problem(sdk_error, "Rust SDK reset required"); + let (sdk_kind, sdk_envelope) = + await_sdk_reset_problem("Rust SDK, first observation", &client, &request); assert_eq!( sdk_kind, "reset_required", "the Rust SDK must classify the terminal as reset required: {sdk_envelope}" @@ -708,36 +762,37 @@ fn reset_required_survives_http_mcp_and_rust_sdk_across_restart() { ); let mount = http_mount(&home_path); - let (_, http_body_after) = post_application( - &mount, - &identity, - STORAGE_STATUS_ROUTE, - &storage_status_body, - None, - ); - super::assert_reset_required( - &problem_envelope(&http_body_after, "HTTP reset required after restart"), - "HTTP mount, after a physical restart", - ); - - let mcp_after = mcp_tool_call( - &home_path, - &project_path, - "tracedecay_storage_status", - &storage_status_body, - None, - ); + let http_problem_after = await_settled_problem("HTTP mount, after a physical restart", || { + let (_, body) = post_application( + &mount, + &identity, + STORAGE_STATUS_ROUTE, + &storage_status_body, + None, + ); + problem_envelope(&body, "HTTP reset required after restart") + }); + super::assert_reset_required(&http_problem_after, "HTTP mount, after a physical restart"); + + let mcp_problem_after = + await_settled_problem("MCP stdio host, after a physical restart", || { + let mcp_after = mcp_tool_call( + &home_path, + &project_path, + "tracedecay_storage_status", + &storage_status_body, + None, + ); + problem_envelope(&mcp_payload(&mcp_after), "MCP reset required after restart") + }); super::assert_reset_required( - &problem_envelope(&mcp_payload(&mcp_after), "MCP reset required after restart"), + &mcp_problem_after, "MCP stdio host, after a physical restart", ); let client = sdk_client(&mount, &identity); - let sdk_error_after = client - .execute::(&request) - .expect_err("a refused store must not read as a healthy status after a restart"); let (sdk_kind_after, sdk_envelope_after) = - sdk_problem(sdk_error_after, "Rust SDK reset required after restart"); + await_sdk_reset_problem("Rust SDK reset required after restart", &client, &request); assert_eq!( sdk_kind_after, "reset_required", "the Rust SDK must keep classifying the terminal as reset required: {sdk_envelope_after}" From 932d880d8b8e7ca00d9c7536f2c30f6f44b61ba2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 05:58:50 +0000 Subject: [PATCH 2/4] style(tests): collapse idle admission check The nested reconciling if trips clippy::collapsible_if under -D warnings. Co-authored-by: Zack Jackson --- .../code_index_ignored_dependencies_test/flight_tests.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs index 77309c5c22..feac73d938 100644 --- a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs +++ b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs @@ -172,10 +172,9 @@ async fn hold_idle_background_admission( loop { if registry.memory_stats().await.reconciling_worktrees == 0 && let Ok(permit) = admission.clone().try_acquire_owned() + && registry.memory_stats().await.reconciling_worktrees == 0 { - if registry.memory_stats().await.reconciling_worktrees == 0 { - return permit; - } + return permit; } assert!( std::time::Instant::now() <= deadline, From e059fef7757bae09e2944193ceff59e6df307f66 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:12:45 +0000 Subject: [PATCH 3/4] test(advisory): wait out accepted_for_replay ingest A completed accepted_for_replay is not a durable commit. Keep polling until committed or exact_duplicate, or the deadline fails closed. Co-authored-by: Zack Jackson --- .../advisory_runtime_acceptance.rs | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/advisory_runtime_acceptance.rs b/crates/tracedecay/tests/runtime_acceptance_suite/advisory_runtime_acceptance.rs index f397d6df9d..d40e739a48 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/advisory_runtime_acceptance.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/advisory_runtime_acceptance.rs @@ -1005,13 +1005,25 @@ async fn packaged_host_ingest_delivers_a_registered_advisory_cycle() { .expect("registered daemon ingest response text"), ) .expect("registered daemon ingest payload"); - if payload["completed"] != false { + // `completed: true` with `accepted_for_replay` means the catch-up + // sweep has not yet drained this admission. That is not a durable + // commit, so keep polling until a terminal that proves the + // transcript, or the deadline reports the last payload. + if matches!( + payload["status"].as_str(), + Some("committed" | "exact_duplicate") + ) { break output; } - assert_eq!( - payload["admission"]["retryable"], true, - "incomplete ingest must carry a retryable admission: {response}" - ); + if payload["completed"] != false && payload["status"] != "accepted_for_replay" { + break output; + } + if payload["completed"] == false { + assert_eq!( + payload["admission"]["retryable"], true, + "incomplete ingest must carry a retryable admission: {response}" + ); + } } else { let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); assert!( From 30dbebab5e9f8c6600a0e2afdb48b71b2fe81556 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 07:00:37 +0000 Subject: [PATCH 4/4] test(code-index): drop the redundant idle-admission hold Master's coalesced_publication_failure fix (0d328fa0a9) takes the generation-store lock before corrupting the active pointer, and its own comment records why the background admission permit is not the proof: the racer releases that permit before it finishes attaching the generation's text artifact, so holding it does not mean the store is quiet. Being granted the store lock does. The extra permit hold this branch layered on top adds a second 5s spin for a race the landed fix already closes. Co-Authored-By: Claude Opus 5 (1M context) --- .../flight_tests.rs | 32 ++----------------- 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs index feac73d938..bc1388df7f 100644 --- a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs +++ b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs @@ -2,7 +2,6 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Condvar, Mutex}; use tracedecay_code_index::production::CodeIndexProductionErrorV1; -use tracedecay_code_index_retention::code_index_generations::try_acquire_code_generation_store_lock; use super::*; @@ -158,32 +157,6 @@ fn assert_publication_error(error: CodeIndexSchedulerErrorV1) { ); } -/// Hold the only background permit once no pass is in flight. -/// -/// Text seating keeps `reconcile_in_progress` after it drops the scheduler -/// mutex, and that pass can still rename a valid active pointer. A truncated -/// pointer written in that window is not a closed fault. Occupying the permit -/// while the owner has not entered its pass stops that rewrite. -async fn hold_idle_background_admission( - registry: &CodeIndexSchedulerRegistryV1, -) -> tokio::sync::OwnedSemaphorePermit { - let admission = registry.background_reconcile_admission(); - let deadline = std::time::Instant::now() + Duration::from_secs(5); - loop { - if registry.memory_stats().await.reconciling_worktrees == 0 - && let Ok(permit) = admission.clone().try_acquire_owned() - && registry.memory_stats().await.reconciling_worktrees == 0 - { - return permit; - } - assert!( - std::time::Instant::now() <= deadline, - "background reconcile did not go idle before publication fault injection" - ); - tokio::time::sleep(Duration::from_millis(2)).await; - } -} - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn aborted_flight_owner_wakes_follower_and_allows_a_fresh_owner() { let fixture = fixture(); @@ -269,8 +242,6 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { let registry = Arc::new(mount(fixture.path(), &store, 1).await); let baseline = latest(®istry, fixture.path()).await; let request = request_for(&baseline, "pkg"); - let idle_admission = hold_idle_background_admission(®istry).await; - registry.clear_pending_wake_for_scope(&request.scope).await; let hold = SchedulerHold::acquire(®istry, fixture.path()).await; let (owner_control, owner_entered) = BlockingNthControl::new(4); @@ -326,6 +297,8 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { // and any writer that starts after it is released reads the corruption // under the lock and refuses instead of overwriting it. let pointer_bytes = { + use tracedecay_code_index_retention::code_index_generations::try_acquire_code_generation_store_lock; + let store_lock = tokio::time::timeout(Duration::from_secs(5), async { loop { if let Some(lock) = try_acquire_code_generation_store_lock(&scoped_store) @@ -343,7 +316,6 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { drop(store_lock); pointer_bytes }; - drop(idle_admission); owner_control.release(); hold.release();