From 7f764338085f0d25fcd1b9d65b2d79971de171d9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 00:06:02 +0000 Subject: [PATCH 1/2] fix(sessions): continue a progressing history window now A Codex catch-up yields after one rollout. Treating that yield as a failure retry inserted a 250ms backoff between every window, so a 33-rollout import never reached Complete inside its 30s deadline. Co-authored-by: Zack Jackson --- .../worker.rs | 68 ++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs index 4384eab2aa..ed7fcde8e0 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs @@ -98,6 +98,30 @@ fn lcm_convergence_admission( /// projection serving continues unblocked. pub(super) const HISTORY_ADMISSION_SATURATED_REASON: &str = "history_admission_saturated"; +/// How the worker schedules the pass after one that still needs history. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum HistoryContinuation { + /// The window admitted work and yielded. Run the next window now. + Immediate, + /// The pass needs another window but admitted nothing. Back off. + Backoff, + /// History does not need another pass. + Settled, +} + +/// A Codex catch-up yields after one rollout. That yield is progress, so the +/// continuation must not pay the no-progress retry delay or a corpus larger +/// than one window misses the import deadline. +fn history_continuation(outcome: Option) -> HistoryContinuation { + match outcome { + Some(SessionHistoricalIngestOutcome::Pending { + made_progress: true, + }) => HistoryContinuation::Immediate, + Some(outcome) if outcome.needs_another_pass() => HistoryContinuation::Backoff, + _ => HistoryContinuation::Settled, + } +} + pub(super) async fn run_session_temporal_refresh_scheduler( database: RegisteredGlobalDbLeaseV1, state: Arc, @@ -386,7 +410,21 @@ pub(super) async fn run_session_temporal_refresh_scheduler( } else if history_needs_another_pass { state.mark_running(); retry_attempt = 0; - state.update_history_retry_state(true); + match history_continuation(history_outcome) { + // A bounded window that admitted work already yielded. The + // next window is continuation of that import, not a failure + // retry: the 250ms backoff below is only for passes that + // made no progress. Sleeping on every successful window + // makes a multi-window corpus miss the import deadline. + HistoryContinuation::Immediate => { + state.update_history_retry_state(false); + state.wake_history(); + } + HistoryContinuation::Backoff => { + state.update_history_retry_state(true); + } + HistoryContinuation::Settled => {} + } } else { if history_outcome.is_some() { state.update_history_retry_state(false); @@ -1104,6 +1142,34 @@ mod tests { assert!(!state.has_pending_work()); } + #[test] + fn a_progressing_history_window_continues_without_the_retry_backoff() { + assert_eq!( + history_continuation(Some(SessionHistoricalIngestOutcome::Pending { + made_progress: true, + })), + HistoryContinuation::Immediate + ); + assert_eq!( + history_continuation(Some(SessionHistoricalIngestOutcome::Pending { + made_progress: false, + })), + HistoryContinuation::Backoff + ); + assert_eq!( + history_continuation(Some(SessionHistoricalIngestOutcome::Retryable { + reason_code: "history_admission_saturated", + made_progress: true, + })), + HistoryContinuation::Backoff, + "a retryable failure keeps the backoff even when the pass wrote rows" + ); + assert_eq!( + history_continuation(Some(SessionHistoricalIngestOutcome::Complete)), + HistoryContinuation::Settled + ); + } + #[test] fn pending_history_windows_take_precedence_over_derived_summaries() { assert!(!history_allows_summary_convergence(Some( From a9c8bffc7bf1825976583f4100646965d7ca3e45 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 00:06:02 +0000 Subject: [PATCH 2/2] test(code-index): sample freshness after the owner settles Text seating keeps reconcile_in_progress after it drops the scheduler mutex, and a follow-up pass can still hold the only admission permit. Both reads then look like a source refresh that the test did not start. Co-authored-by: Zack Jackson --- .../src/code_index_scheduler/tests/mod.rs | 36 ++++++++++ .../code_index_scheduler/tests/reconcile.rs | 65 ++++++++++++++----- 2 files changed, 85 insertions(+), 16 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs index 3b6ba3ceb6..0935aee090 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs @@ -1288,6 +1288,42 @@ async fn quiesced_background_reconcile_admission( admission } +/// Hold background admission only once no pass and no wake are outstanding. +/// +/// [`quiesced_background_reconcile_admission`] can win the permit while text +/// seating still owns `reconcile_in_progress`: that seating is a real refresh, +/// and a freshness read then reports `Verifying` even though the scheduler +/// mutex is free. A caller that must attribute `Verifying` to something other +/// than an unrelated mutex holder has to sample after both that guard and any +/// coalesced wake are clear, and keep the permit so a successor cannot start +/// under the sample. +async fn hold_settled_background_admission( + registry: &CodeIndexSchedulerRegistryV1, + project_root: &Path, +) -> tokio::sync::OwnedSemaphorePermit { + let deadline = Instant::now() + SERVING_SEAT_FAILURE_CEILING; + loop { + let admission = registry + .background_reconcile_admission() + .acquire_owned() + .await + .expect("hold background worker at its dequeue point"); + wait_for_quiescent_owner_pass(registry, project_root).await; + let pending_wake = registry.pending_wake_micros_for_root(project_root).await; + let in_progress = registry.reconcile_in_progress_for_test(project_root).await; + if pending_wake == Some(0) && !in_progress { + return admission; + } + drop(admission); + assert!( + Instant::now() <= deadline, + "the owner for {} never settled before a freshness sample", + project_root.display() + ); + wait_for_settled_owner(registry, project_root).await; + } +} + const CALLER_STAR: usize = 2_000; const CALLER_STAR_FILES: usize = 8; const CALLER_PAGE: u32 = 10; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs index 41254144b5..84b7c4b0c6 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs @@ -25,16 +25,17 @@ use tracedecay_runtime_core::resident_memory::{ use super::{ ALPHA_LIB_V1, GitFixture, RETAINED_REVISION_0, SERVING_SEAT_FAILURE_CEILING, advance_pointer_to_unseated_successor, application_context, committed_capture_corpus_files, - core_search_request, drain_clone_backfill, git, git_stdout, mounted_core_query_worktree, - mounted_core_query_worktree_with_one_permit, published, query_authority, query_meta, - quiesced_background_reconcile_admission, replace_scheduler_chunker_revision, - replace_scheduler_policy_revision, rewrite_active_rust_extractor_revision, - rewrite_preserving_stat, scheduler, scheduler_with_policy, served_lexical_texts, - test_project_id, wait_for_dashboard_ready, wait_for_event_to_ready, wait_for_generation_change, - wait_for_initial_generation, wait_for_live_complete_generation, - wait_for_live_complete_generation_by_polling, wait_for_queryable_text_generation, - wait_for_queryable_text_generation_change, wait_for_queryable_text_generation_id, - wait_for_quiescent_owner_pass, wait_for_settled_owner, wait_until_serving_seat, write, + core_search_request, drain_clone_backfill, git, git_stdout, hold_settled_background_admission, + mounted_core_query_worktree, mounted_core_query_worktree_with_one_permit, published, + query_authority, query_meta, quiesced_background_reconcile_admission, + replace_scheduler_chunker_revision, replace_scheduler_policy_revision, + rewrite_active_rust_extractor_revision, rewrite_preserving_stat, scheduler, + scheduler_with_policy, served_lexical_texts, test_project_id, wait_for_dashboard_ready, + wait_for_event_to_ready, wait_for_generation_change, wait_for_initial_generation, + wait_for_live_complete_generation, wait_for_live_complete_generation_by_polling, + wait_for_queryable_text_generation, wait_for_queryable_text_generation_change, + wait_for_queryable_text_generation_id, wait_for_quiescent_owner_pass, wait_for_settled_owner, + wait_until_serving_seat, write, }; use crate::{ code_index::{ @@ -2899,13 +2900,37 @@ async fn ignored_dependency_waits_for_global_admission_before_publication_gate() ) }; let global_admission = registry.background_reconcile_admission(); + // `drain_clone_backfill` drops the permit it held, and the worker may + // already be inside the pass that follows. That pass owns the only + // permit, so the assertion below is a race unless setup waits for the + // worker to release it — and drops this gate if the worker is blocked on + // it — before the request under test is admitted. registry.clear_pending_wake_for_scope(&scope).await; - let publication = publication_gate.lock().await; - assert_eq!( - global_admission.available_permits(), - 1, - "test setup requires an idle global admission" - ); + let idle_deadline = Instant::now() + SERVING_SEAT_FAILURE_CEILING; + let publication = loop { + while global_admission.available_permits() == 0 + || registry + .reconcile_in_progress_for_test(fixture.path()) + .await + { + assert!( + Instant::now() <= idle_deadline, + "background worker must release global admission before the publication gate is held" + ); + tokio::time::sleep(Duration::from_millis(2)).await; + } + registry.clear_pending_wake_for_scope(&scope).await; + let publication = publication_gate.lock().await; + let available = global_admission.available_permits(); + if available == 1 { + break publication; + } + drop(publication); + assert!( + Instant::now() <= idle_deadline, + "test setup requires an idle global admission" + ); + }; let request_registry = registry.clone(); let project_root = fixture.path().to_path_buf(); @@ -3681,6 +3706,10 @@ async fn dashboard_progress_does_not_wait_for_the_scheduler_mutex() { .expect("mount daemon-owned scheduler"); wait_for_initial_generation(®istry, fixture.path()).await; wait_for_dashboard_ready(®istry, fixture.path()).await; + // Clone backfill posts its own wakes after the seat. Drain them before + // the progress sample so a successor pass cannot look like this test's + // unrelated scheduler-mutex holder. + drain_clone_backfill(®istry, fixture.path()).await; let canonical_root = fixture .path() .canonicalize() @@ -3703,6 +3732,10 @@ async fn dashboard_progress_does_not_wait_for_the_scheduler_mutex() { }) .await .expect("background text build publishes progress"); + // Text seating keeps `reconcile_in_progress` after it releases the + // scheduler mutex. Sampling in that window reports Verifying for a real + // pass, which is not this test's unrelated holder. + let _admission = hold_settled_background_admission(®istry, fixture.path()).await; let (locked_tx, locked_rx) = tokio::sync::oneshot::channel(); let (release_tx, release_rx) = tokio::sync::oneshot::channel();