From 690e84365d875954488298e7f07d835d67325dc2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 06:25:49 +0000 Subject: [PATCH 1/4] fix(retention): defer a census that races a missing path A generation-retention census that does not hold the store lock can observe a scope root the publisher has not created, or a file a peer unlinked after it was listed. That NotFound is the same deferral as a held writer, not a storage failure. The mounted journey waits until the superseded source is actually collectable. Co-authored-by: Zack Jackson --- .../src/code_index_generations.rs | 21 ++++++++-- .../code_index_generations/generation_scan.rs | 7 ++-- .../src/code_index_generations/locking.rs | 4 +- .../src/code_index_generations/tests.rs | 35 ++++++++++++++++ .../generation_retention_test.rs | 42 +++++++++++++++---- 5 files changed, 92 insertions(+), 17 deletions(-) diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations.rs b/crates/tracedecay-code-index-retention/src/code_index_generations.rs index b4976c6602..78c68296bd 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations.rs @@ -885,7 +885,7 @@ fn plan_code_generation_retention_with_verification_cancellable( Err(error) if error.kind() == std::io::ErrorKind::NotFound && active_pointer.is_none() => { None } - Err(error) => return Err(storage(error)), + Err(error) => return Err(deferred_if_absent(error)), }; let mut generations = BTreeMap::new(); let mut active_state_digest = None; @@ -1218,7 +1218,7 @@ fn sweep_unreferenced_generation_segments( continue; } let mut reader = CancellableGenerationManifestReaderV1 { - file: File::open(&path).map_err(storage)?, + file: File::open(&path).map_err(deferred_if_absent)?, hasher: Sha256::new(), is_cancelled, cancelled: false, @@ -1281,7 +1281,7 @@ fn sweep_unreferenced_generation_segments( if live_segments.contains(&format!("sha256:{digest}")) { continue; } - let metadata = path.symlink_metadata().map_err(storage)?; + let metadata = path.symlink_metadata().map_err(deferred_if_absent)?; if !metadata.file_type().is_file() { return Err(CodeGenerationRetentionErrorV1::UnsafeState(format!( "generation segment '{}' is not a regular file", @@ -1745,7 +1745,7 @@ fn read_active_pointer( store_root: &Path, ) -> Result { let path = store_root.join(ACTIVE_POINTER_FILE); - let bytes = std::fs::read(&path).map_err(storage)?; + let bytes = std::fs::read(&path).map_err(deferred_if_absent)?; serde_json::from_slice(&bytes).map_err(|error| { CodeGenerationRetentionErrorV1::UnsafeState(format!( "active pointer '{}' is corrupt: {error}", @@ -2033,5 +2033,18 @@ fn storage(error: impl std::fmt::Display) -> CodeGenerationRetentionErrorV1 { CodeGenerationRetentionErrorV1::Storage(error.to_string()) } +/// A path that is not there yet, or that a peer unlinked after this census +/// listed it, is not a broken disk. The publisher creates the scope root and +/// the sealed files under the store lock, then drops that lock; a census that +/// does not hold the lock can observe the gap. The next tick sees a stable +/// tree. Every other I/O failure stays a storage error. +pub(super) fn deferred_if_absent(error: std::io::Error) -> CodeGenerationRetentionErrorV1 { + if error.kind() == std::io::ErrorKind::NotFound { + CodeGenerationRetentionErrorV1::GenerationStoreBusy + } else { + storage(error) + } +} + #[cfg(test)] mod tests; diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/generation_scan.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/generation_scan.rs index 91b997173c..7c2923eaf2 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/generation_scan.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/generation_scan.rs @@ -7,7 +7,8 @@ use tracedecay_domain::canonical_text::{encode_tagged_lowercase_hex, is_lowercas use super::{ CodeGenerationRetentionErrorV1, GenerationDigestVerificationV1, - MAX_GENERATION_METADATA_PREFIX_BYTES, SealedGenerationManifestMetadataV1, storage, + MAX_GENERATION_METADATA_PREFIX_BYTES, SealedGenerationManifestMetadataV1, deferred_if_absent, + storage, }; const MAX_FORMAT_REVISION_PREFIX_BYTES: usize = 4 * 1024; @@ -16,7 +17,7 @@ pub(super) fn read_generation_format_revision( path: &Path, is_cancelled: &dyn Fn() -> bool, ) -> Result { - let mut file = File::open(path).map_err(storage)?; + let mut file = File::open(path).map_err(deferred_if_absent)?; let mut prefix = vec![0_u8; MAX_FORMAT_REVISION_PREFIX_BYTES]; let bytes_read = file.read(&mut prefix).map_err(storage)?; crate::hotpath_observe::retention_inspected(bytes_read as u64); @@ -40,7 +41,7 @@ pub(super) fn read_generation_metadata( is_cancelled: &dyn Fn() -> bool, ) -> Result<(u32, SealedGenerationManifestMetadataV1, String, u64), CodeGenerationRetentionErrorV1> { - let mut file = File::open(path).map_err(storage)?; + let mut file = File::open(path).map_err(deferred_if_absent)?; let size_bytes = file.metadata().map_err(storage)?.len(); let mut hasher = Sha256::new(); let mut prefix = Vec::with_capacity(MAX_GENERATION_METADATA_PREFIX_BYTES); diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/locking.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/locking.rs index 6bdc552abd..1d8d867a75 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/locking.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/locking.rs @@ -102,7 +102,7 @@ fn lock_file( } fn canonical_store_root(store_root: &Path) -> Result { - std::fs::canonicalize(store_root).map_err(storage) + std::fs::canonicalize(store_root).map_err(super::deferred_if_absent) } fn open_lock_file(path: &Path) -> Result { @@ -112,5 +112,5 @@ fn open_lock_file(path: &Path) -> Result { .write(true) .truncate(false) .open(path) - .map_err(storage) + .map_err(super::deferred_if_absent) } diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs index 2dbe91a0c7..116a8becc6 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs @@ -1501,6 +1501,41 @@ fn idle_maintenance_preparation_stays_metadata_only() { ); } +#[test] +fn preparation_defers_when_the_scope_root_does_not_exist_yet() { + let parent = tempfile::TempDir::new().expect("parent"); + let missing = parent.path().join("not-created"); + let error = prepare_next_code_generation_retention_cancellable( + &missing, + &BTreeSet::new(), + &|| false, + None, + ) + .expect_err("an unpublished scope root has no census"); + assert!( + matches!(error, CodeGenerationRetentionErrorV1::GenerationStoreBusy), + "a missing scope root is the publisher's create window, not a storage failure: {error:?}" + ); +} + +#[test] +fn preparation_defers_when_the_pointer_exists_without_its_generation_directory() { + let (store, _generations) = fixture_store(1); + std::fs::remove_dir_all(store.path().join(GENERATIONS_DIRECTORY)) + .expect("remove generation directory under a live pointer"); + let error = prepare_next_code_generation_retention_cancellable( + store.path(), + &BTreeSet::new(), + &|| false, + None, + ) + .expect_err("the generation directory is not durable yet"); + assert!( + matches!(error, CodeGenerationRetentionErrorV1::GenerationStoreBusy), + "a pointer without its generation directory is a torn publish, not a storage failure: {error:?}" + ); +} + #[test] fn metadata_only_segment_census_observes_at_most_one_directory_entry() { let store = tempfile::TempDir::new().expect("create unpublished store"); diff --git a/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs b/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs index b0d224dfec..0d8f8dd83d 100644 --- a/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs @@ -11,7 +11,8 @@ use super::journey_test_support::git; use super::*; use crate::daemon::maintenance::project_store_maintenance_lease; use tracedecay_code_index_retention::code_index_generations::{ - MAX_CODE_GENERATION_RETENTION_BATCH_V1, prepare_next_code_generation_retention_cancellable, + CodeGenerationRetentionErrorV1, MAX_CODE_GENERATION_RETENTION_BATCH_V1, + prepare_next_code_generation_retention_cancellable, }; use tracedecay_maintenance::tick::{MaintenanceContinuation, MaintenanceTickOutcome}; @@ -114,13 +115,38 @@ async fn mounted_code_generation_retention_continues_capped_segment_reclamation( &canonical_root, ); let graph_replay_pool_root = graph.db().database_path().with_extension("graph-replay"); - let plan = prepare_next_code_generation_retention_cancellable( - &code_store_root, - &BTreeSet::new(), - &|| false, - Some(&graph_replay_pool_root), - ) - .expect("code generation retention plan"); + // The serving id moves when the swap installs the generation. The sealed + // files and the replay pool are still being published and retired beside + // that swap, so one census can miss the scope root or a file it just + // listed. Those reads are `GenerationStoreBusy`, not a failed journey. + let plan = tokio::time::timeout(Duration::from_secs(20), async { + loop { + match prepare_next_code_generation_retention_cancellable( + &code_store_root, + &BTreeSet::new(), + &|| false, + Some(&graph_replay_pool_root), + ) { + Ok(plan) + if plan + .collectable_generations + .iter() + .any(|generation| generation.generation_id == first_source) => + { + return plan; + } + Ok(_) + | Err( + CodeGenerationRetentionErrorV1::GenerationStoreBusy + | CodeGenerationRetentionErrorV1::GraphReplayPoolBusy, + ) => {} + Err(error) => panic!("code generation retention plan: {error:?}"), + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("superseded source became collectable"); let first_candidate = plan .collectable_generations .iter() From 151836457716b30cde4c7009f13d081ed2c49022 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 07:00:09 +0000 Subject: [PATCH 2/4] test(retention): assert a vanished census open defers Opening a listed generation that publication already unlinked must not be Storage. The census defers as store-busy; a non-NotFound open stays storage. Co-authored-by: Zack Jackson --- .../src/code_index_generations/tests.rs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs index 116a8becc6..f8e99ab2bf 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs @@ -3107,3 +3107,28 @@ fn recovery_completes_a_committed_rewrite_that_never_reached_the_pointer() { plan_code_generation_retention(fixture.store.path(), &BTreeSet::new()) .expect("a recovered store must stay plannable"); } + +/// The census opens every name `read_dir` just returned. Publication can +/// unlink that name first. `NotFound` is the same deferral as a held writer, +/// not a storage failure. Any other open failure stays storage. +#[test] +fn vanished_listed_generation_open_defers_instead_of_storage_loss() { + let root = tempfile::tempdir().expect("census root"); + let missing = root.path().join(format!("generation-{:064x}.json", 1)); + let error = super::generation_scan::read_generation_format_revision(&missing, &|| false) + .expect_err("a vanished listed generation defers the census"); + assert!( + matches!(error, CodeGenerationRetentionErrorV1::GenerationStoreBusy), + "a missing listed generation is a publisher race, not a storage failure: {error:?}" + ); + + let directory = root.path().join("not-a-generation-file"); + std::fs::create_dir(&directory).expect("directory where a file was listed"); + let storage_error = + super::generation_scan::read_generation_format_revision(&directory, &|| false) + .expect_err("a directory is not a vanished file"); + assert!( + matches!(storage_error, CodeGenerationRetentionErrorV1::Storage(_)), + "non-NotFound census I/O stays a storage failure: {storage_error:?}" + ); +} From d6d8665a80068bf8b28a995cdff6c412997720be Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 08:34:45 +0000 Subject: [PATCH 3/4] fix(retention): keep absent durable state a storage failure The merged branch reclassified every `NotFound` in the generation census as `GenerationStoreBusy`. Two of those sites are not publisher races: - The generations directory read under a live active pointer. The sealer writes the sealed file, its directory and the pointer under one store-lock hold, so a durable pointer implies a durable directory; an absent directory beside a live pointer is loss. Production consumes `GenerationStoreBusy` with `defer_generation_store_busy`, so deferring here turns that loss into a silent per-tick defer that never reclaims and never reports degraded. - `read_active_pointer`. Its lock-free caller `read_optional_active_pointer` already stats the pointer and owns the typed unpublished answer, and the pointer is installed by atomic rename so it is never transiently absent; its other caller, `mutate_verified_text_artifact_under_lock`, holds the store lock and compares against an expected pointer. The enumerate-then-open sites keep the deferral: the census lists a directory without the store lock, so a name it just read can be unlinked before the open, which is the race the branch set out to fix. Drops the unit test that asserted the reverted classification. Co-Authored-By: Claude Fable 5.1 --- .../src/code_index_generations.rs | 7 +++++-- .../src/code_index_generations/tests.rs | 18 ------------------ 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations.rs b/crates/tracedecay-code-index-retention/src/code_index_generations.rs index 78c68296bd..b5f76713c9 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations.rs @@ -885,7 +885,10 @@ fn plan_code_generation_retention_with_verification_cancellable( Err(error) if error.kind() == std::io::ErrorKind::NotFound && active_pointer.is_none() => { None } - Err(error) => return Err(deferred_if_absent(error)), + // A pointer is only durable once its generation directory is, so a + // live pointer over an absent directory is loss, not a publisher + // race, and must stay loud. + Err(error) => return Err(storage(error)), }; let mut generations = BTreeMap::new(); let mut active_state_digest = None; @@ -1745,7 +1748,7 @@ fn read_active_pointer( store_root: &Path, ) -> Result { let path = store_root.join(ACTIVE_POINTER_FILE); - let bytes = std::fs::read(&path).map_err(deferred_if_absent)?; + let bytes = std::fs::read(&path).map_err(storage)?; serde_json::from_slice(&bytes).map_err(|error| { CodeGenerationRetentionErrorV1::UnsafeState(format!( "active pointer '{}' is corrupt: {error}", diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs index 87baa7ad01..47d6d406f9 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs @@ -1564,24 +1564,6 @@ fn preparation_defers_when_the_scope_root_does_not_exist_yet() { ); } -#[test] -fn preparation_defers_when_the_pointer_exists_without_its_generation_directory() { - let (store, _generations) = fixture_store(1); - std::fs::remove_dir_all(store.path().join(GENERATIONS_DIRECTORY)) - .expect("remove generation directory under a live pointer"); - let error = prepare_next_code_generation_retention_cancellable( - store.path(), - &BTreeSet::new(), - &|| false, - None, - ) - .expect_err("the generation directory is not durable yet"); - assert!( - matches!(error, CodeGenerationRetentionErrorV1::GenerationStoreBusy), - "a pointer without its generation directory is a torn publish, not a storage failure: {error:?}" - ); -} - #[test] fn metadata_only_segment_census_observes_at_most_one_directory_entry() { let store = tempfile::TempDir::new().expect("create unpublished store"); From b5231fd649060b542b99f601080fb51aeae57e88 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 08:22:00 +0000 Subject: [PATCH 4/4] style(session-temporal): keep the test module last and rustfmt master Master run 35431539771 failed Check formatting (projector.rs, query.rs) and Clippy (items_after_test_module in query.rs) after #1844/#1845 merged without CI. Co-Authored-By: Claude Fable 5.1 --- .../projector.rs | 8 ++--- .../src/query.rs | 29 +++++++++---------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs index 10818733c1..632898ca24 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/projector.rs @@ -119,11 +119,11 @@ impl SessionTemporalRefreshProjector for CanonicalSessionTemporalProjector { // Empty remaining range is a durable no-op: terminalize with an // empty complete progress batch instead of deferring forever. Ok(None) => canonical_noop_complete_effect(&recovery), - Err(error) if error.is_storage() => Err( - SessionTemporalRefreshProjectorError::retryable(format!( + Err(error) if error.is_storage() => { + Err(SessionTemporalRefreshProjectorError::retryable(format!( "source_busy: {error}" - )), - ), + ))) + } Err(_) => Err(SessionTemporalRefreshProjectorError::terminal( "projector_failed", )), diff --git a/crates/tracedecay-session-temporal-store/src/query.rs b/crates/tracedecay-session-temporal-store/src/query.rs index 367b14bb79..ae0ab6b4de 100644 --- a/crates/tracedecay-session-temporal-store/src/query.rs +++ b/crates/tracedecay-session-temporal-store/src/query.rs @@ -246,8 +246,7 @@ pub(super) async fn read_observations( // ceiling. Split until a single observation remains; that // observation is then a typed storage failure, not a retry // that looks like a busy source. - if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 - { + if observation_prefetch_exceeded_materialization_limit(&error) && chunk.len() > 1 { let mid = start + chunk.len() / 2; pending.push((mid, end)); pending.push((start, mid)); @@ -289,11 +288,18 @@ fn observation_prefetch_exceeded_materialization_limit(error: &SessionStoreError } } +/// The error `read_observation` raises for an id the store does not hold, reused +/// by callers that resolve prefetched observations out of a batch map. +pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { + storage_message( + PERSIST_OPERATION, + format!("source observation {} is missing", observation_id.as_str()), + ) +} + #[cfg(test)] mod tests { - use super::{ - PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage, - }; + use super::{PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage}; #[test] fn materialization_limit_is_the_prefetch_split_signal() { @@ -308,15 +314,8 @@ mod tests { assert!(observation_prefetch_exceeded_materialization_limit( &exceeded )); - assert!(!observation_prefetch_exceeded_materialization_limit(&locked)); + assert!(!observation_prefetch_exceeded_materialization_limit( + &locked + )); } } - -/// The error `read_observation` raises for an id the store does not hold, reused -/// by callers that resolve prefetched observations out of a batch map. -pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> SessionStoreError { - storage_message( - PERSIST_OPERATION, - format!("source observation {} is missing", observation_id.as_str()), - ) -}