From 0a9f666725105ef9033df67b6eb32b4920e363e9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:19:35 +0000 Subject: [PATCH 1/5] fix(daemon): start open publication bound at claim The 500 ms wait answered how long the connection had been here, so route enrollment could spend the budget before the request joined the open. A restart then reported warming for a store the open had already refused with reset_required. The bound now starts at the claim, and the waiter reads a recorded refusal before the cache probe a deadline can cancel. Co-authored-by: Zack Jackson --- crates/tracedecay/src/daemon.rs | 12 +++++ crates/tracedecay/src/daemon/engine.rs | 15 +++++- .../src/daemon/project_open_orchestration.rs | 26 +++++++--- .../tracedecay/src/daemon/tests/bootstrap.rs | 50 +++++++++++++++---- 4 files changed, 83 insertions(+), 20 deletions(-) diff --git a/crates/tracedecay/src/daemon.rs b/crates/tracedecay/src/daemon.rs index 13f830244b..7a6746506c 100644 --- a/crates/tracedecay/src/daemon.rs +++ b/crates/tracedecay/src/daemon.rs @@ -77,6 +77,18 @@ const MAX_CACHED_PROJECT_SERVERS: usize = 8; const MAX_TRACKED_PROJECT_OPEN_TASKS: usize = MAX_CACHED_PROJECT_SERVERS; const MAX_CACHED_PROJECT_OPEN_FAILURES: usize = 64; const PROJECT_OPEN_REQUEST_DEADLINE: Duration = Duration::from_millis(500); + +/// Instant a foreground request stops waiting for an open it has already claimed. +/// +/// Measured from the claim, not from connection arrival. Route enrollment and +/// git discovery have their own bounds. Charging them to this deadline made a +/// restart's first request answer warming for a store whose open had already +/// recorded `reset_required`: the connection spent the budget before it joined +/// the watch, so the publication wait returned immediately and never read the +/// refusal. +fn project_open_publication_deadline(claimed_at: tokio::time::Instant) -> tokio::time::Instant { + claimed_at + PROJECT_OPEN_REQUEST_DEADLINE +} /// One budget for every blocking repository probe a route resolution runs. /// /// Route resolution reads the repository's topology, enrollment marker, and diff --git a/crates/tracedecay/src/daemon/engine.rs b/crates/tracedecay/src/daemon/engine.rs index 30f76cdc89..26d746eb16 100644 --- a/crates/tracedecay/src/daemon/engine.rs +++ b/crates/tracedecay/src/daemon/engine.rs @@ -680,15 +680,28 @@ impl DaemonEngine { // warm-up runs. The open task remains tracked and continues in the // background after this bounded wait expires. let mut retry_init = handshake.allow_init; - let publication_deadline = tokio::time::Instant::now() + PROJECT_OPEN_REQUEST_DEADLINE; loop { let claim = Box::pin(self.begin_project_open(handshake.clone(), None)).await?; + // The bound starts here, after the open is claimed. Starting it + // at connection arrival let route enrollment spend it, and the + // request then answered warming for a refusal already on the watch. + let publication_deadline = + project_open_publication_deadline(tokio::time::Instant::now()); let result = match claim { ProjectOpenTaskClaim::InFlight(state) => { let recorded = state.clone(); let publication = async { let mut state = state; loop { + // A recorded refusal is the route's answer. Read it + // before the cache probe: that probe is an await, and + // an elapsed bound cancels it, which is how a + // connection reported warming for `reset_required`. + if let ProjectOpenTaskState::Failed(failure) = + state.borrow().clone() + { + return Err(failure.to_error()); + } // The claim proves an open for this exact route is // in flight, so each iteration only needs to see // its publication land on the already-bound route diff --git a/crates/tracedecay/src/daemon/project_open_orchestration.rs b/crates/tracedecay/src/daemon/project_open_orchestration.rs index 98745f7194..a8354368b8 100644 --- a/crates/tracedecay/src/daemon/project_open_orchestration.rs +++ b/crates/tracedecay/src/daemon/project_open_orchestration.rs @@ -17,13 +17,13 @@ pub(super) async fn wait_for_project_open_publication( where Publication: std::future::Future>, { - // The bound is a plain deadline: a waiter resumed after it elapsed still - // needs one more await, `route_bound_project_server`, before its - // publication loop can read the route's terminal state, so an elapsed - // deadline preempts a failure that was already recorded and the caller - // would see warming for a route that is no longer opening. Callers repair - // that with `prefer_recorded_open_failure` against the claim's own watch - // channel instead of weakening the bound. + // The bound is a plain deadline, measured from the open claim. A waiter + // resumed after it elapsed may still be inside `route_bound_project_server` + // and would otherwise answer warming for a refusal already on the watch. + // Callers repair that with `prefer_recorded_open_failure` against the + // claim's own watch channel instead of weakening the bound. The publication + // loop also reads `Failed` before that await, so a poll that sees the + // refusal returns it even when the deadline is also ready. hotpath::future!( tokio::time::timeout_at(deadline, publication), label = "daemon.project.open.publication_wait" @@ -483,7 +483,6 @@ pub(super) async fn portable_project_server_for_request( // warm-up runs. The open task remains tracked and continues in the // background after this bounded wait expires. let mut retry_init = handshake.allow_init; - let publication_deadline = tokio::time::Instant::now() + PROJECT_OPEN_REQUEST_DEADLINE; loop { let claim = Box::pin(begin_portable_project_open( lifecycle.clone(), @@ -499,12 +498,23 @@ pub(super) async fn portable_project_server_for_request( project_open_attempts.clone(), )) .await; + // The bound starts here, after the open is claimed. Starting it at + // connection arrival let route enrollment spend it, and the request + // then answered warming for a refusal already on the watch. + let publication_deadline = project_open_publication_deadline(tokio::time::Instant::now()); let result = match claim { ProjectOpenTaskClaim::InFlight(state) => { let recorded = state.clone(); let publication = async { let mut state = state; loop { + // A recorded refusal is the route's answer. Read it + // before the cache probe: that probe is an await, and + // an elapsed bound cancels it, which is how a + // connection reported warming for `reset_required`. + if let ProjectOpenTaskState::Failed(failure) = state.borrow().clone() { + return Err(failure.to_error()); + } if let Some(server) = portable_cached_project_server( &store_administration, &canonical_project_path, diff --git a/crates/tracedecay/src/daemon/tests/bootstrap.rs b/crates/tracedecay/src/daemon/tests/bootstrap.rs index 7426df8ba7..e014b63b94 100644 --- a/crates/tracedecay/src/daemon/tests/bootstrap.rs +++ b/crates/tracedecay/src/daemon/tests/bootstrap.rs @@ -2249,10 +2249,9 @@ async fn explicit_init_retries_after_joining_an_ordinary_missing_database_open() let _database_scope = enter_test_daemon_database_scope(&profile_root, "registered missing-db init retry"); let engine = test_daemon_engine_for_profile(&profile_root); - // Each request arms its publication bound on its first poll, before route - // enrollment resolves. Opening and migrating the profile database on that - // path would spend the whole bound before either request subscribes to the - // open below, so the runtime is warmed the way daemon bootstrap warms it. + // Profile open sits before the publication bound (the bound starts when + // the open is claimed). Warm it so the 150 ms join window below is spent + // on the controlled open, not on the first profile migration. prewarm_test_profile_runtime(&engine.store_administration).await; let ordinary_handshake = DaemonHandshake { project_path: Some(project.clone()), @@ -2264,13 +2263,10 @@ async fn explicit_init_retries_after_joining_an_ordinary_missing_database_open() allow_init: true, ..ordinary_handshake.clone() }; - // The bound also covers everything a request does *before* it can join an - // open, route enrollment, git discovery, the registered layout, so a - // first-touch request can spend the whole bound before it ever subscribes - // to the open below and would then mint its own. One ordinary request - // ahead of the fixture resolves that path for this exact route, and its - // refusal is the same missing-index failure the joined waiter classifies - // later. + // One ordinary request ahead of the fixture resolves this exact route, + // and its refusal is the same missing-index failure the joined waiter + // classifies later. The publication bound no longer includes that + // enrollment, so the request can subscribe before the bound is spent. // It is retried the way a client retries the warming hint, so the route is // left with no open of its own before the fixture takes it over. let warmup_give_up = tokio::time::Instant::now() + std::time::Duration::from_secs(30); @@ -3561,6 +3557,38 @@ async fn foreground_project_open_wait_is_bounded_and_accepts_quick_publication() ); } +/// Route enrollment that consumes the old connection-arrival budget must not +/// turn a quick `reset_required` into warming. +/// +/// The publication future is pending on its first poll, which is exactly when +/// an already-elapsed bound used to win: the waiter never read the refusal. +/// The bound is measured from the claim, so enrollment time is not part of it. +#[tokio::test(start_paused = true)] +async fn enrollment_delay_does_not_hide_a_quick_reset_required() { + tokio::time::advance(super::super::PROJECT_OPEN_REQUEST_DEADLINE).await; + let deadline = super::super::project_open_publication_deadline(tokio::time::Instant::now()); + let refused = super::super::project_open_orchestration::wait_for_project_open_publication( + std::path::Path::new("/projects/reset-required"), + deadline, + async { + tokio::task::yield_now().await; + Err(tracedecay_domain::errors::TraceDecayError::reset_required( + "project store", + "database schema contains unexpected table", + )) + }, + ) + .await + .expect_err("a refused store must not publish a server"); + assert!( + matches!( + refused, + tracedecay_domain::errors::TraceDecayError::ResetRequired { .. } + ), + "enrollment that spent the connection-arrival budget must not answer warming: {refused}" + ); +} + fn production_composition_tool_text(response: &JsonRpcResponse) -> &str { assert!(response.error.is_none(), "tool failed: {response:?}"); let result = response.result.as_ref().expect("tool result"); From bf0eeb460405e5c1a63db4bab5aeae0b61f4a145 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:44:44 +0000 Subject: [PATCH 2/5] test(daemon): type the reset publication future The paused enrollment-delay check needs an explicit output type so the publication wait can see the reset_required refusal. Co-authored-by: Zack Jackson --- crates/tracedecay/src/daemon/tests/bootstrap.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay/src/daemon/tests/bootstrap.rs b/crates/tracedecay/src/daemon/tests/bootstrap.rs index e014b63b94..b320a8413d 100644 --- a/crates/tracedecay/src/daemon/tests/bootstrap.rs +++ b/crates/tracedecay/src/daemon/tests/bootstrap.rs @@ -3572,10 +3572,12 @@ async fn enrollment_delay_does_not_hide_a_quick_reset_required() { deadline, async { tokio::task::yield_now().await; - Err(tracedecay_domain::errors::TraceDecayError::reset_required( - "project store", - "database schema contains unexpected table", - )) + Err::<(), tracedecay_domain::errors::TraceDecayError>( + tracedecay_domain::errors::TraceDecayError::reset_required( + "project store", + "database schema contains unexpected table", + ), + ) }, ) .await From e3860bf9c199177a3e4bd2f53e3edfaa6b23001a Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 19:36:48 +0000 Subject: [PATCH 3/5] fix(daemon): keep a published owner ahead of a recorded failure The pre-cache `Failed` read made a waiter answer with its own claim's recorded failure even when another open had since published a server for the same route, contradicting the invariant `a_recorded_open_failure_replaces_only_the_warming_hint` states. The read was also redundant. With the bound measured from the claim, the publication wait always starts with a full budget, so an elapsed deadline can only preempt a later iteration, and `prefer_recorded_open_failure` already substitutes the recorded failure for that warming hint. Co-Authored-By: Claude Opus 5 (1M context) --- crates/tracedecay/src/daemon/engine.rs | 9 --------- .../src/daemon/project_open_orchestration.rs | 11 +---------- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/crates/tracedecay/src/daemon/engine.rs b/crates/tracedecay/src/daemon/engine.rs index 26d746eb16..37725cc976 100644 --- a/crates/tracedecay/src/daemon/engine.rs +++ b/crates/tracedecay/src/daemon/engine.rs @@ -693,15 +693,6 @@ impl DaemonEngine { let publication = async { let mut state = state; loop { - // A recorded refusal is the route's answer. Read it - // before the cache probe: that probe is an await, and - // an elapsed bound cancels it, which is how a - // connection reported warming for `reset_required`. - if let ProjectOpenTaskState::Failed(failure) = - state.borrow().clone() - { - return Err(failure.to_error()); - } // The claim proves an open for this exact route is // in flight, so each iteration only needs to see // its publication land on the already-bound route diff --git a/crates/tracedecay/src/daemon/project_open_orchestration.rs b/crates/tracedecay/src/daemon/project_open_orchestration.rs index a8354368b8..3f1426f0e5 100644 --- a/crates/tracedecay/src/daemon/project_open_orchestration.rs +++ b/crates/tracedecay/src/daemon/project_open_orchestration.rs @@ -21,9 +21,7 @@ where // resumed after it elapsed may still be inside `route_bound_project_server` // and would otherwise answer warming for a refusal already on the watch. // Callers repair that with `prefer_recorded_open_failure` against the - // claim's own watch channel instead of weakening the bound. The publication - // loop also reads `Failed` before that await, so a poll that sees the - // refusal returns it even when the deadline is also ready. + // claim's own watch channel instead of weakening the bound. hotpath::future!( tokio::time::timeout_at(deadline, publication), label = "daemon.project.open.publication_wait" @@ -508,13 +506,6 @@ pub(super) async fn portable_project_server_for_request( let publication = async { let mut state = state; loop { - // A recorded refusal is the route's answer. Read it - // before the cache probe: that probe is an await, and - // an elapsed bound cancels it, which is how a - // connection reported warming for `reset_required`. - if let ProjectOpenTaskState::Failed(failure) = state.borrow().clone() { - return Err(failure.to_error()); - } if let Some(server) = portable_cached_project_server( &store_administration, &canonical_project_path, From 6bee62a76a7b784f7b1638faf2221755e92db70d Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 19:36:49 +0000 Subject: [PATCH 4/5] test(daemon): drop the tautological enrollment-bound test `enrollment_delay_does_not_hide_a_quick_reset_required` computed its own deadline with `project_open_publication_deadline(now)`, so it never observed where the request loop arms the bound. Reverting the production change to master's connection-arrival placement left the test passing, which is the detection it claimed to provide. `reset_required_survives_http_mcp_and_rust_sdk_across_restart` remains the detector for this defect. Co-Authored-By: Claude Opus 5 (1M context) --- .../tracedecay/src/daemon/tests/bootstrap.rs | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/crates/tracedecay/src/daemon/tests/bootstrap.rs b/crates/tracedecay/src/daemon/tests/bootstrap.rs index b320a8413d..c2c1d7c7af 100644 --- a/crates/tracedecay/src/daemon/tests/bootstrap.rs +++ b/crates/tracedecay/src/daemon/tests/bootstrap.rs @@ -3557,40 +3557,6 @@ async fn foreground_project_open_wait_is_bounded_and_accepts_quick_publication() ); } -/// Route enrollment that consumes the old connection-arrival budget must not -/// turn a quick `reset_required` into warming. -/// -/// The publication future is pending on its first poll, which is exactly when -/// an already-elapsed bound used to win: the waiter never read the refusal. -/// The bound is measured from the claim, so enrollment time is not part of it. -#[tokio::test(start_paused = true)] -async fn enrollment_delay_does_not_hide_a_quick_reset_required() { - tokio::time::advance(super::super::PROJECT_OPEN_REQUEST_DEADLINE).await; - let deadline = super::super::project_open_publication_deadline(tokio::time::Instant::now()); - let refused = super::super::project_open_orchestration::wait_for_project_open_publication( - std::path::Path::new("/projects/reset-required"), - deadline, - async { - tokio::task::yield_now().await; - Err::<(), tracedecay_domain::errors::TraceDecayError>( - tracedecay_domain::errors::TraceDecayError::reset_required( - "project store", - "database schema contains unexpected table", - ), - ) - }, - ) - .await - .expect_err("a refused store must not publish a server"); - assert!( - matches!( - refused, - tracedecay_domain::errors::TraceDecayError::ResetRequired { .. } - ), - "enrollment that spent the connection-arrival budget must not answer warming: {refused}" - ); -} - fn production_composition_tool_text(response: &JsonRpcResponse) -> &str { assert!(response.error.is_none(), "tool failed: {response:?}"); let result = response.result.as_ref().expect("tool result"); From 49ad17d9c4d90e48922156829814bacd9882fbaf Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 19:49:44 +0000 Subject: [PATCH 5/5] chore(daemon): drop a comment that restates the deadline doc Co-Authored-By: Claude Fable 5.1 --- crates/tracedecay/src/daemon/engine.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/crates/tracedecay/src/daemon/engine.rs b/crates/tracedecay/src/daemon/engine.rs index 37725cc976..b629ba31bd 100644 --- a/crates/tracedecay/src/daemon/engine.rs +++ b/crates/tracedecay/src/daemon/engine.rs @@ -682,9 +682,6 @@ impl DaemonEngine { let mut retry_init = handshake.allow_init; loop { let claim = Box::pin(self.begin_project_open(handshake.clone(), None)).await?; - // The bound starts here, after the open is claimed. Starting it - // at connection arrival let route enrollment spend it, and the - // request then answered warming for a refusal already on the watch. let publication_deadline = project_open_publication_deadline(tokio::time::Instant::now()); let result = match claim {