From d86fd22b4901d4b209894142fb50683ae673d1a5 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 08:53:16 +0000 Subject: [PATCH] fix(code-index): never join clone backfill from a freshness read A live blocked status request was waiting in clone_index_status on the clone successor slot mutex. Backfill holds that lock for a whole bounded batch, so status, dashboard freshness, unmounted-files, and doctor reads could stall for 30-120 seconds behind production work. Read the slot with try_lock instead; a busy slot returns the typed unavailable clone-status snapshot immediately. The dashboard-level regression holds the slot and requires that result within 100 ms. --- .../src/code_index_scheduler/serving.rs | 36 ++++++++++++---- .../src/code_index_scheduler/tests/serving.rs | 42 +++++++++++++++++++ 2 files changed, 71 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs index 4a0c14c1cd..814b5a6a6f 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs @@ -1657,6 +1657,15 @@ impl LatestCodeTextGenerationV1 { }; } }; + let successor = match self.clone_successor_progress() { + CloneSuccessorProgressReadV1::Idle => None, + CloneSuccessorProgressReadV1::Backfilling(progress) => Some(progress), + CloneSuccessorProgressReadV1::Busy => { + return CodeCloneIndexStatusV1::Unavailable { + reason: "clone-index status is being updated".to_owned(), + }; + } + }; let artifact = match owners.clone_index_artifact() { Ok(artifact) => artifact, Err(error) => { @@ -1665,7 +1674,6 @@ impl LatestCodeTextGenerationV1 { }; } }; - let successor = self.clone_successor_progress(); let (completed_source_pages, total_source_pages, bytes_on_disk) = successor.map_or( ( artifact.source_pages, @@ -1709,16 +1717,24 @@ impl LatestCodeTextGenerationV1 { } } - fn clone_successor_progress(&self) -> Option { - let slot = self.text_projection_build.lock_slot(); + fn clone_successor_progress(&self) -> CloneSuccessorProgressReadV1 { + let slot = match self.text_projection_build.slot.try_lock() { + Ok(slot) => slot, + Err(std::sync::TryLockError::WouldBlock) => { + return CloneSuccessorProgressReadV1::Busy; + } + Err(std::sync::TryLockError::Poisoned(poisoned)) => poisoned.into_inner(), + }; match &*slot { CodeTextProjectionSlotV1::CloneSuccessorPending => { let owners = match self.query_owner_readiness() { CodeTextQueryOwnerReadinessV1::Ready(owners) => owners, CodeTextQueryOwnerReadinessV1::Pending - | CodeTextQueryOwnerReadinessV1::Invalid => return None, + | CodeTextQueryOwnerReadinessV1::Invalid => { + return CloneSuccessorProgressReadV1::Idle; + } }; - Some(CloneSuccessorProgressV1 { + CloneSuccessorProgressReadV1::Backfilling(CloneSuccessorProgressV1 { completed_source_pages: 0, total_source_pages: owners.hydration.verified_artifact().page_count(), bytes_on_disk: None, @@ -1730,7 +1746,7 @@ impl LatestCodeTextGenerationV1 { .as_ref() .and_then(|builder| builder.next_cursor().ok().flatten()) .map_or(0, |cursor| cursor.next_page_ordinal()); - Some(CloneSuccessorProgressV1 { + CloneSuccessorProgressReadV1::Backfilling(CloneSuccessorProgressV1 { completed_source_pages, total_source_pages: build.prior.page_count(), bytes_on_disk: build @@ -1742,7 +1758,7 @@ impl LatestCodeTextGenerationV1 { } CodeTextProjectionSlotV1::Idle | CodeTextProjectionSlotV1::HeadOpening - | CodeTextProjectionSlotV1::Building(_) => None, + | CodeTextProjectionSlotV1::Building(_) => CloneSuccessorProgressReadV1::Idle, } } @@ -1762,6 +1778,12 @@ struct CloneSuccessorProgressV1 { bytes_on_disk: Option, } +enum CloneSuccessorProgressReadV1 { + Idle, + Backfilling(CloneSuccessorProgressV1), + Busy, +} + fn clone_index_observation( text: &LatestCodeTextGenerationV1, artifact: &CloneIndexArtifactSnapshotV1, diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs index 7f0fdf658f..19bba01627 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs @@ -912,6 +912,48 @@ fn clone_status_distinguishes_unavailable_backfill_partial_ready_and_stale() { )); } +#[tokio::test] +async fn dashboard_freshness_does_not_join_a_clone_backfill_slice() { + let fixture = GitFixture::new(&[( + "src/lib.rs", + "pub fn alpha() { one(); two(); three(); four(); five(); six(); seven(); eight(); nine(); ten(); }\n", + )]); + let store = TempDir::new().expect("store root"); + let registry = CodeIndexSchedulerRegistryV1::new(1); + registry + .mount_worktree( + test_project_id(), + fixture.path(), + store.path().to_path_buf(), + ) + .await + .expect("mount worktree"); + let latest = wait_for_queryable_text_generation(®istry, fixture.path()).await; + while !latest.query_owners_are_ready() { + latest.advance_text_serving(1).expect("advance text build"); + } + + let held_slot = latest.text_projection_build.lock_slot(); + let freshness = tokio::time::timeout( + Duration::from_millis(100), + registry.dashboard_freshness(fixture.path()), + ) + .await + .expect("dashboard freshness must not wait for the clone backfill slice") + .expect("mounted dashboard freshness"); + assert!(matches!( + freshness.clone_index, + Some( + tracedecay_contracts::code_index_freshness::CodeCloneIndexStatusV1::Unavailable { + reason + } + ) if reason == "clone-index status is being updated" + )); + + drop(held_slot); + registry.shutdown().await; +} + #[tokio::test] async fn query_admission_serves_v14_while_clone_successor_is_pending() { let fixture = GitFixture::new(&[(