Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1288,6 +1288,42 @@ async fn quiesced_background_reconcile_admission(
admission
}

/// Hold background admission only once no pass and no wake are outstanding.
///
/// [`quiesced_background_reconcile_admission`] can win the permit while text
/// seating still owns `reconcile_in_progress`: that seating is a real refresh,
/// and a freshness read then reports `Verifying` even though the scheduler
/// mutex is free. A caller that must attribute `Verifying` to something other
/// than an unrelated mutex holder has to sample after both that guard and any
/// coalesced wake are clear, and keep the permit so a successor cannot start
/// under the sample.
async fn hold_settled_background_admission(
registry: &CodeIndexSchedulerRegistryV1,
project_root: &Path,
) -> tokio::sync::OwnedSemaphorePermit {
let deadline = Instant::now() + SERVING_SEAT_FAILURE_CEILING;
loop {
let admission = registry
.background_reconcile_admission()
.acquire_owned()
.await
.expect("hold background worker at its dequeue point");
wait_for_quiescent_owner_pass(registry, project_root).await;
let pending_wake = registry.pending_wake_micros_for_root(project_root).await;
let in_progress = registry.reconcile_in_progress_for_test(project_root).await;
if pending_wake == Some(0) && !in_progress {
return admission;
}
drop(admission);
assert!(
Instant::now() <= deadline,
"the owner for {} never settled before a freshness sample",
project_root.display()
);
wait_for_settled_owner(registry, project_root).await;
}
}

const CALLER_STAR: usize = 2_000;
const CALLER_STAR_FILES: usize = 8;
const CALLER_PAGE: u32 = 10;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,17 @@ use tracedecay_runtime_core::resident_memory::{
use super::{
ALPHA_LIB_V1, GitFixture, RETAINED_REVISION_0, SERVING_SEAT_FAILURE_CEILING,
advance_pointer_to_unseated_successor, application_context, committed_capture_corpus_files,
core_search_request, drain_clone_backfill, git, git_stdout, mounted_core_query_worktree,
mounted_core_query_worktree_with_one_permit, published, query_authority, query_meta,
quiesced_background_reconcile_admission, replace_scheduler_chunker_revision,
replace_scheduler_policy_revision, rewrite_active_rust_extractor_revision,
rewrite_preserving_stat, scheduler, scheduler_with_policy, served_lexical_texts,
test_project_id, wait_for_dashboard_ready, wait_for_event_to_ready, wait_for_generation_change,
wait_for_initial_generation, wait_for_live_complete_generation,
wait_for_live_complete_generation_by_polling, wait_for_queryable_text_generation,
wait_for_queryable_text_generation_change, wait_for_queryable_text_generation_id,
wait_for_quiescent_owner_pass, wait_for_settled_owner, wait_until_serving_seat, write,
core_search_request, drain_clone_backfill, git, git_stdout, hold_settled_background_admission,
mounted_core_query_worktree, mounted_core_query_worktree_with_one_permit, published,
query_authority, query_meta, quiesced_background_reconcile_admission,
replace_scheduler_chunker_revision, replace_scheduler_policy_revision,
rewrite_active_rust_extractor_revision, rewrite_preserving_stat, scheduler,
scheduler_with_policy, served_lexical_texts, test_project_id, wait_for_dashboard_ready,
wait_for_event_to_ready, wait_for_generation_change, wait_for_initial_generation,
wait_for_live_complete_generation, wait_for_live_complete_generation_by_polling,
wait_for_queryable_text_generation, wait_for_queryable_text_generation_change,
wait_for_queryable_text_generation_id, wait_for_quiescent_owner_pass, wait_for_settled_owner,
wait_until_serving_seat, write,
};
use crate::{
code_index::{
Expand Down Expand Up @@ -2899,13 +2900,37 @@ async fn ignored_dependency_waits_for_global_admission_before_publication_gate()
)
};
let global_admission = registry.background_reconcile_admission();
// `drain_clone_backfill` drops the permit it held, and the worker may
// already be inside the pass that follows. That pass owns the only
// permit, so the assertion below is a race unless setup waits for the
// worker to release it — and drops this gate if the worker is blocked on
// it — before the request under test is admitted.
registry.clear_pending_wake_for_scope(&scope).await;
let publication = publication_gate.lock().await;
assert_eq!(
global_admission.available_permits(),
1,
"test setup requires an idle global admission"
);
let idle_deadline = Instant::now() + SERVING_SEAT_FAILURE_CEILING;
let publication = loop {
while global_admission.available_permits() == 0
|| registry
.reconcile_in_progress_for_test(fixture.path())
.await
{
assert!(
Instant::now() <= idle_deadline,
"background worker must release global admission before the publication gate is held"
);
tokio::time::sleep(Duration::from_millis(2)).await;
}
registry.clear_pending_wake_for_scope(&scope).await;
let publication = publication_gate.lock().await;
let available = global_admission.available_permits();
if available == 1 {
break publication;
Comment on lines +2924 to +2926

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Ensure the request owns the observed admission permit

When a background Notify permit remains after clear_pending_wake_for_scope, the worker can wake after this check, acquire the sole semaphore permit, and block on the held publication gate. The later wait for available_permits() == 0 then succeeds even though request_task has not acquired admission, because the mount worker acquires admission before the publication lock and only sets reconcile_in_progress afterward. Thus the test can falsely pass for the ordering regression it is meant to detect; synchronize on the request's acquisition or otherwise distinguish the permit owner.

AGENTS.md reference: AGENTS.md:L165-L167

Useful? React with 👍 / 👎.

}
drop(publication);
assert!(
Instant::now() <= idle_deadline,
"test setup requires an idle global admission"
);
};

let request_registry = registry.clone();
let project_root = fixture.path().to_path_buf();
Expand Down Expand Up @@ -3681,6 +3706,10 @@ async fn dashboard_progress_does_not_wait_for_the_scheduler_mutex() {
.expect("mount daemon-owned scheduler");
wait_for_initial_generation(&registry, fixture.path()).await;
wait_for_dashboard_ready(&registry, fixture.path()).await;
// Clone backfill posts its own wakes after the seat. Drain them before
// the progress sample so a successor pass cannot look like this test's
// unrelated scheduler-mutex holder.
drain_clone_backfill(&registry, fixture.path()).await;
let canonical_root = fixture
.path()
.canonicalize()
Expand All @@ -3703,6 +3732,10 @@ async fn dashboard_progress_does_not_wait_for_the_scheduler_mutex() {
})
.await
.expect("background text build publishes progress");
// Text seating keeps `reconcile_in_progress` after it releases the
// scheduler mutex. Sampling in that window reports Verifying for a real
// pass, which is not this test's unrelated holder.
let _admission = hold_settled_background_admission(&registry, fixture.path()).await;

let (locked_tx, locked_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,30 @@ fn lcm_convergence_admission(
/// projection serving continues unblocked.
pub(super) const HISTORY_ADMISSION_SATURATED_REASON: &str = "history_admission_saturated";

/// How the worker schedules the pass after one that still needs history.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum HistoryContinuation {
/// The window admitted work and yielded. Run the next window now.
Immediate,
/// The pass needs another window but admitted nothing. Back off.
Backoff,
/// History does not need another pass.
Settled,
}

/// A Codex catch-up yields after one rollout. That yield is progress, so the
/// continuation must not pay the no-progress retry delay or a corpus larger
/// than one window misses the import deadline.
fn history_continuation(outcome: Option<SessionHistoricalIngestOutcome>) -> HistoryContinuation {
match outcome {
Some(SessionHistoricalIngestOutcome::Pending {
made_progress: true,
}) => HistoryContinuation::Immediate,
Some(outcome) if outcome.needs_another_pass() => HistoryContinuation::Backoff,
_ => HistoryContinuation::Settled,
}
}

pub(super) async fn run_session_temporal_refresh_scheduler(
database: RegisteredGlobalDbLeaseV1,
state: Arc<SessionTemporalRefreshWakeState>,
Expand Down Expand Up @@ -386,7 +410,21 @@ pub(super) async fn run_session_temporal_refresh_scheduler(
} else if history_needs_another_pass {
state.mark_running();
retry_attempt = 0;
state.update_history_retry_state(true);
match history_continuation(history_outcome) {
// A bounded window that admitted work already yielded. The
// next window is continuation of that import, not a failure
// retry: the 250ms backoff below is only for passes that
// made no progress. Sleeping on every successful window
// makes a multi-window corpus miss the import deadline.
HistoryContinuation::Immediate => {
state.update_history_retry_state(false);
state.wake_history();
}
HistoryContinuation::Backoff => {
state.update_history_retry_state(true);
}
HistoryContinuation::Settled => {}
}
} else {
if history_outcome.is_some() {
state.update_history_retry_state(false);
Expand Down Expand Up @@ -1104,6 +1142,34 @@ mod tests {
assert!(!state.has_pending_work());
}

#[test]
fn a_progressing_history_window_continues_without_the_retry_backoff() {
assert_eq!(
history_continuation(Some(SessionHistoricalIngestOutcome::Pending {
made_progress: true,
})),
HistoryContinuation::Immediate
);
assert_eq!(
history_continuation(Some(SessionHistoricalIngestOutcome::Pending {
made_progress: false,
})),
HistoryContinuation::Backoff
);
assert_eq!(
history_continuation(Some(SessionHistoricalIngestOutcome::Retryable {
reason_code: "history_admission_saturated",
made_progress: true,
})),
HistoryContinuation::Backoff,
"a retryable failure keeps the backoff even when the pass wrote rows"
);
assert_eq!(
history_continuation(Some(SessionHistoricalIngestOutcome::Complete)),
HistoryContinuation::Settled
);
}

#[test]
fn pending_history_windows_take_precedence_over_derived_summaries() {
assert!(!history_allows_summary_convergence(Some(
Expand Down
Loading