From 191cc62fdc788e0ce581f0202fbfc1f9d838f4fc Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 18:07:39 +0000 Subject: [PATCH 01/19] ci: fix rustfmt drift, clippy doc lint, bench readiness slack - cargo fmt drift in code_index_generations/locking.rs and lifecycle_lease.rs - clippy::doc_lazy_continuation in tracedecay-privacy rules.rs (a prose pass had split a sentence across a lazy continuation and left a ////// line) - benchmark harness: the dashboard HTTP-variant self-test gave the stub 0.2s to bind, which on a loaded runner timed out in dashboard_connect Co-Authored-By: Claude Fable 5.1 --- benchmark_data/runtime/tests/test_lifecycle.py | 4 +++- .../src/code_index_generations/locking.rs | 5 +---- crates/tracedecay-privacy/src/rules.rs | 8 ++++---- .../src/lifecycle_lease.rs | 15 +++------------ 4 files changed, 11 insertions(+), 21 deletions(-) diff --git a/benchmark_data/runtime/tests/test_lifecycle.py b/benchmark_data/runtime/tests/test_lifecycle.py index 9ffdbe3398..0792ec32fe 100644 --- a/benchmark_data/runtime/tests/test_lifecycle.py +++ b/benchmark_data/runtime/tests/test_lifecycle.py @@ -160,7 +160,9 @@ def test_dashboard_http_variants_remain_typed_failures(self) -> None: url, request_timeout=0.05, ), - readiness_timeout=0.2, + # The stub takes a moment to bind on a loaded runner; + # 0.2s left the probe stuck in dashboard_connect. + readiness_timeout=2.0, poll_interval=0.01, termination_grace=0.05, ) diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/locking.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/locking.rs index 8d53fed465..6bdc552abd 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/locking.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/locking.rs @@ -47,10 +47,7 @@ pub fn try_acquire_code_generation_store_read_lock( ) -> Result, CodeGenerationRetentionErrorV1> { let store_root = canonical_store_root(store_root)?; let lock = open_lock_file(&store_root.join(STORE_LOCK_FILE))?; - match lock - .try_lock_shared() - .map_err(std::io::Error::from) - { + match lock.try_lock_shared().map_err(std::io::Error::from) { Ok(()) => Ok(Some(CodeGenerationStoreLockV1 { file: lock, store_root, diff --git a/crates/tracedecay-privacy/src/rules.rs b/crates/tracedecay-privacy/src/rules.rs index 131c2d3f15..605ae0eac6 100644 --- a/crates/tracedecay-privacy/src/rules.rs +++ b/crates/tracedecay-privacy/src/rules.rs @@ -654,10 +654,10 @@ fn compile_regex( /// so it is *both* a different match and vastly larger to compile: three /// upstream rules that repeat `\w` over a wide bound /// (`pypi-...[\w-]{50,1000}`) blow past the compiler's 10 MB program limit. -/// Expanding `\w` to its RE2 meaning fixes the semantics and the size at once -/// , every rule in the catalogue then compiles under the default limit, with -/// no memory headroom bought and no rule dropped. -////// * **`\b` / `\B`.** RE2's word boundary is ASCII. Rust's is Unicode-aware, +/// Expanding `\w` to its RE2 meaning fixes the semantics and the size at +/// once: every rule in the catalogue then compiles under the default limit, +/// with no memory headroom bought and no rule dropped. +/// * **`\b` / `\B`.** RE2's word boundary is ASCII. Rust's is Unicode-aware, /// and a Unicode boundary is the one construct the lazy DFA gives up on the /// moment the haystack holds a non-ASCII byte: every file with an em-dash or /// an emoji in a comment was then scanned by the PikeVM, the slowest engine, diff --git a/crates/tracedecay-runtime-core/src/lifecycle_lease.rs b/crates/tracedecay-runtime-core/src/lifecycle_lease.rs index d7a88ae33d..05b95672bd 100644 --- a/crates/tracedecay-runtime-core/src/lifecycle_lease.rs +++ b/crates/tracedecay-runtime-core/src/lifecycle_lease.rs @@ -221,10 +221,7 @@ pub fn acquire_shared_or_inherited(operation: &str) -> Result { fn acquire_shared_or_inherited_at(path: &Path, operation: &str) -> Result { let mut file = open_lock_file(path)?; - match file - .try_lock_shared() - .map_err(std::io::Error::from) - { + match file.try_lock_shared().map_err(std::io::Error::from) { Ok(()) => Ok(LifecycleLease { hold: LeaseHold::File(file), token: None, @@ -384,10 +381,7 @@ fn acquire_exclusive_at_with_timeout( #[hotpath::measure(label = "runtime_core.lifecycle.acquire_shared")] fn acquire_shared_at(path: &Path, operation: &str) -> Result { let mut file = open_lock_file(path)?; - match file - .try_lock_shared() - .map_err(std::io::Error::from) - { + match file.try_lock_shared().map_err(std::io::Error::from) { Ok(()) => Ok(LifecycleLease { hold: LeaseHold::File(file), token: None, @@ -404,10 +398,7 @@ fn acquire_shared_at(path: &Path, operation: &str) -> Result { fn try_acquire_shared_at(path: &Path, operation: &str) -> Result { let file = open_lock_file(path)?; - match file - .try_lock_shared() - .map_err(std::io::Error::from) - { + match file.try_lock_shared().map_err(std::io::Error::from) { Ok(()) => Ok(SharedLeaseAttempt::Acquired(LifecycleLease { hold: LeaseHold::File(file), token: None, From acc2773e9021bd10a9ae19894a179bb7684bee3b Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 18:58:03 +0000 Subject: [PATCH 02/19] test(code-index): re-pin partitioned codec onto extractor.rust.v10 The pinned partitioned manifest bytes were measured at extractor.rust.v8. Two revision bumps have landed since: fe58bba408 ("move every extractor revision for the clone-body bound") took Rust to v9, and 941d908ba9 ("bound body bytes before tokenizing large literals") took it to v10. The revision string is part of every sealed file segment, so the state digest and all four segment digests move. v8 and v9 are the same length but v10 is one character longer, which is the whole size change: each of the three file segments grows by exactly one byte (11_070 -> 11_071, 5_170 -> 5_171, 6_278 -> 6_279) and the evidence segment, which carries no per-file extractor revision, stays at 6_837. Each file segment was confirmed to hold `extractor.rust.v10` exactly once. format_revision is unchanged at 12 (set by e358627c60, before the last pin), so this is an intentional extractor identity change, not container drift or recording noise. Co-Authored-By: Claude Fable 5.1 --- .../code_index_suite/production_orchestration.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs index 151889363d..f5beb287d7 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs @@ -3244,22 +3244,22 @@ fn partitioned_codec_fixture() -> ( } const PARTITIONED_FORMAT_STATE_DIGEST: &str = - "sha256:28f30287a415e81bf589922385146f921a539734ec0a3600bd39578ad3c8dcd3"; + "sha256:8d84348830efc4452a078cfac1cc78e0ed44112a37f1025bd6f4f4bc152fe196"; const PARTITIONED_FORMAT_SEGMENTS: &[(&str, u64)] = &[ ( - "sha256:924c1f0b7b171b7bf5433a6eb04767eb244e7c9decc1f2343b06a657908f3a7b", - 11_070, + "sha256:e50d2733b5f594d79fdccc3e44b5d30d5efb66d14805b5fe0c67d5ceb0a1d66f", + 11_071, ), ( - "sha256:1a6e240c8fcc1084d82cee42cbaa889bb479ff1df7a76432dcec2d51d66e1d1a", - 5_170, + "sha256:1095d61bb8bbbf6637f85ca957a510d221aaba7923af8e60b0f3eef07042e6ff", + 5_171, ), ( - "sha256:4461a4ce08e5f59299030959a2773bd48b2c2d48056c188e06867db99609847a", - 6_278, + "sha256:9921ca7da5c489307887ab570a5e8d5a7cebf192b9b6664c487e9a327943e396", + 6_279, ), ( - "sha256:4c54bba48f3fcf2fd0085ab5aecd8b35451a8cfc56381fa99605327165afbb15", + "sha256:52b5707b5312bcb1e29849372b0dbb882205b3289b345643620a35c1d260c246", 6_837, ), ]; From 508424b02a8760565137e8ee2bd24abda70ecf30 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 18:58:15 +0000 Subject: [PATCH 03/19] test(code-index): measure legacy restore against a paged control The guard compared the legacy restore's absolute peak-RSS growth against half the on-disk generation, treating the paged restore that runs first as a warm-up that leaves the allocator holding the arena a restored generation needs. That only holds when the allocator keeps the freed arena. On the CI runner it returns the pages, so the legacy probe faults them in again and its "extra" cost is the whole restored generation: paged grew 4276 KiB and legacy 4768 KiB, and the 4882432-byte legacy figure failed a bound of half the 8641409-byte generation. The 492 KiB that actually separated the two forms was far below the 1736866-byte evidence segment, so nothing was being materialized. Both forms issue the identical reads (300 whole file reads and 7 ranged evidence reads): load_next_legacy_chunk streams an unpaged descriptor in GENERATION_EVIDENCE_PAGE_MAX_BYTES_V1 chunks, so the production path is correct and only the measurement was wrong. Measure the difference instead. One discarded probe pays the process's cold-start cost, then the paged restore of the same generation is the control and the legacy restore's growth beyond it is the pre-paging path's own cost. That cost must stay below the evidence segment, which a materializing restore would hold whole. Verified by injecting a retained copy of every evidence page: the guard fires at 1.37x the segment, while the streaming restore measures 0.12x to 0.38x. The bound is tighter than the one it replaces, which permitted 2.5x this segment. Co-Authored-By: Claude Fable 5.1 --- .../production_orchestration.rs | 43 +++++++++++++------ 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs index f5beb287d7..c07cdca96a 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs @@ -4601,11 +4601,11 @@ hwm_delta_kib={hwm_delta}" /// The shipped restore read the whole segment, parsed a `serde_json::Value` /// from it, rewrote identities in that tree and deserialized the tree again: /// peak memory was 2.35x the on-disk generation and grew with the corpus. The -/// paged restore of the same generation runs first here, so the allocator -/// already holds the arena a restored generation needs; the legacy restore's -/// own peak growth over that baseline is therefore the extra cost of the -/// pre-paging path alone, and it must stay far below the generation's on-disk -/// size rather than scaling with it. +/// paged restore of the same generation runs first here as the control: both +/// forms pay the restored generation's own memory, so the legacy restore's +/// peak growth beyond the control is the extra cost of the pre-paging path +/// alone, and it must stay far below the evidence segment rather than +/// scaling with it. #[test] fn legacy_generation_restore_does_not_materialize_its_evidence_segment() { const RSS_CHILD: &str = "TD_LEGACY_RSS_CHILD"; @@ -4696,22 +4696,39 @@ fn legacy_generation_restore_does_not_materialize_its_evidence_segment() { drop(generation); drop(owner); + // The first restore in a process pays a cold-start cost (the arena a + // restored generation needs) that has nothing to do with the form being + // restored. Spend it on a discarded probe so the two measured probes + // start from the same allocator state and stay comparable. + rss_measure_decode("warmup", &paged_manifest, &segments); let paged_hwm = rss_measure_decode("paged", &paged_manifest, &segments); let legacy_hwm = rss_measure_decode("legacy", &legacy_manifest, &segments); - let legacy_bytes = legacy_hwm * 1024; + // The paged decode of the same generation is the control, not a warm-up: + // both forms restore the identical generation, so only the difference is + // the pre-paging path's own cost. An absolute peak is not a usable + // measure, it is dominated by whether the allocator returned the control + // decode's pages to the OS between the two probes, which a developer box + // and a CI runner answer differently by more than the segment under test. + let legacy_extra_bytes = legacy_hwm.saturating_sub(paged_hwm) * 1024; println!( "rss_summary files={file_count} generation_on_disk_bytes={generation_bytes} \ evidence_segment_bytes={evidence_bytes} paged_hwm_delta_kib={paged_hwm} \ -legacy_hwm_delta_kib={legacy_hwm} legacy_over_generation={:.3} legacy_over_evidence={:.3}", - legacy_bytes as f64 / generation_bytes as f64, - legacy_bytes as f64 / evidence_bytes as f64, +legacy_hwm_delta_kib={legacy_hwm} legacy_extra_bytes={legacy_extra_bytes} \ +legacy_extra_over_evidence={:.3}", + legacy_extra_bytes as f64 / evidence_bytes as f64, ); + // A restore that materializes the segment holds all of it at once and + // parses it on top, so it costs at least the segment; the streaming + // restore costs one bounded page buffer plus allocator slack, measured + // at a fifth to a third of the segment. The bound sits between them, and + // is tighter than the half-the-whole-generation bound it replaces (which + // permitted two and a half times this segment). assert!( - legacy_bytes * 2 < generation_bytes as u64, - "restoring a pre-paging generation grew peak RSS by {legacy_bytes} bytes over a warmed \ - baseline, which is not far below the {generation_bytes}-byte on-disk generation \ - ({evidence_bytes}-byte evidence segment): the segment is being materialized" + legacy_extra_bytes < evidence_bytes as u64, + "restoring a pre-paging generation cost {legacy_extra_bytes} bytes of peak RSS beyond the \ + paged restore of the same {generation_bytes}-byte generation, which is not far below its \ + {evidence_bytes}-byte evidence segment: the segment is being materialized" ); } From 4350c09eb27fdea6e7f04f562bb3acd5d3d7e865 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 18:58:27 +0000 Subject: [PATCH 04/19] test(agent-hosts): pin Codex CLI presence for the prepare probe 7bdc33d3cb ("clear LCM, proxy, codex, partial, rebuild reds") made prepare_non_interactive_install return a typed DeferredUserAction when Codex's own plugin CLI is absent, because Core apply can only drive `codex plugin add` when the host binary is there. The outcome therefore became a property of the machine: the test passed on a developer box with `codex` on PATH and failed on CI runners, which carry none. Install an executable `codex` on the host-program search path for the duration of the test, the same seam the Gemini and Kiro lifecycle tests use. Only host program resolution sees the fixture directory; the process PATH is untouched, and the guard's own lock keeps a second override from being installed while this one is held. The stub is never executed, preparation only asks whether the binary resolves. Verified hermetic by running the built test binary with a PATH that has no `codex` on it. Co-Authored-By: Claude Fable 5.1 --- .../src/agents/codex/tests.rs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/crates/tracedecay-agent-hosts/src/agents/codex/tests.rs b/crates/tracedecay-agent-hosts/src/agents/codex/tests.rs index b146a5e75e..1a6f1b1c9a 100644 --- a/crates/tracedecay-agent-hosts/src/agents/codex/tests.rs +++ b/crates/tracedecay-agent-hosts/src/agents/codex/tests.rs @@ -979,9 +979,34 @@ fn codex_preflight_reports_inactive_cache_without_interactive_guidance() { assert!(CodexIntegration.interactive_removal_guidance().is_none()); } +/// Install an executable `codex` on the host-program search path only. +/// +/// Preparation is `Ready` exactly when Codex's own plugin CLI is present, so +/// the outcome under test is a property of the environment, not of the host +/// integration. CI runners carry no `codex` binary while a developer box +/// usually does; pin it here instead of reading whichever the machine has. +/// Only host program resolution sees this directory, the process `PATH` is +/// untouched. +fn install_fake_codex_cli( + dir: &Path, +) -> tracedecay_runtime_core::config::HostProgramSearchPathGuard { + let binary = dir.join(format!("codex{}", std::env::consts::EXE_SUFFIX)); + std::fs::write(&binary, "#!/bin/sh\nexit 0\n").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut permissions = std::fs::metadata(&binary).unwrap().permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(&binary, permissions).unwrap(); + } + tracedecay_runtime_core::config::HostProgramSearchPathGuard::set(dir) +} + #[test] fn prepare_stages_the_source_and_returns_ready_for_cli_activation() { let home = tempfile::tempdir().unwrap(); + let cli_dir = tempfile::tempdir().unwrap(); + let _codex_cli = install_fake_codex_cli(cli_dir.path()); // Pre-existing user config: preparation runs before the component // transaction stages `config.toml`, so it must not write there, hook // trust is recorded by activation, inside the rollback boundary. From b344238ad05824f5e150a80d5ee41e779b90e17a Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 18:58:28 +0000 Subject: [PATCH 05/19] test(dashboard-api): match the credential gate step case-insensitively d73b3150a8 ("drop em dashes and stock phrasing from prose") rewrote the GitHubCredentialNotConfigured reason from "... for this profile and repository - configure a token (or register ...)" to two sentences, so the clause now starts with a capital: "Configure a token (or register ...)". The guidance the gate contracts to carry is intact; only the sentence break moved, and the case-sensitive substring check broke on it. Compare case-insensitively. The contract is that the gate names the step the reader has to take, not where a sentence happens to break around it. Co-Authored-By: Claude Fable 5.1 --- crates/tracedecay-dashboard-api/src/delivery_api.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-dashboard-api/src/delivery_api.rs b/crates/tracedecay-dashboard-api/src/delivery_api.rs index df715d0121..88b2b9d2c6 100644 --- a/crates/tracedecay-dashboard-api/src/delivery_api.rs +++ b/crates/tracedecay-dashboard-api/src/delivery_api.rs @@ -2474,7 +2474,9 @@ mod tests { panic!("a gated mount must project as typed unavailable"); }; assert!( - reason.contains("configure a token"), + // Case-insensitive: the contract is that the gate names the step, + // not where the sentence happens to break around it. + reason.to_ascii_lowercase().contains("configure a token"), "the credential gate must tell the reader what to do: {reason}" ); From 7bcefef14f99ce7a1a658194aaf6a699bb625eef Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 19:09:59 +0000 Subject: [PATCH 06/19] test(search-eval): re-pin query-fallback receipts after revision bumps The packaged query-fallback receipts drifted once every extractor revision moved for the clone-body bound: c7eb62eaea (PR #1741) took Rust v8 to v9, TypeScript/protobuf/SQL v4 to v5 and the rest v3 to v4, and 542d28c4c0 (PR #1784) took Rust to v10. The extractor revision is part of the extraction batch identity, so the sealed generation every candidate binding names moves with it, and the fallback subpayload digest hashes those bindings. Bisecting the packaged comparison over the range since the last re-pin (3320ad48d4) lands on c7eb62eaea, whose only production change is those three revision integers. The ranking did not change. Comparing the packaged report at 3320ad48d4 against the current tip, every per-query row is byte identical: same first useful rank, returned candidates, wrong-scope and forbidden hits, and quality. Both partitions keep their exact conceptual-miss sets (10 train, 7 validation) and their mean reciprocal rank (476881 ppm train, 587222 ppm validation). Only the provenance identity inside the receipt moved. Re-pin both partition receipts, packaged::WORKLOAD_SHA256, and the byte-pinned workload digest both search-eval bins assert. Co-Authored-By: Claude Fable 5.1 --- .../search_quality/query-lexical-graph-workload-v1.json | 4 ++-- crates/tracedecay-query/src/search_quality/packaged.rs | 2 +- .../src/bin/tracedecay-search-eval-direct.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json b/crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json index 864c0e0578..4646d75e25 100644 --- a/crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json +++ b/crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json @@ -135,8 +135,8 @@ } ], "expected_query_fallback_digests": { - "train": "sha256:d020b61b1658145487d4d1ea9622ad6075293c329e202bb351c49290bc398067", - "validation": "sha256:934459d069cc368f06ce81d950116d4c823965a95c8f752eef9d41b45a0c01cb" + "train": "sha256:5750d4a588f7a7e14c381ec3a4285400a29e164babbf88e677ce7441aaf8e9b2", + "validation": "sha256:e7a459efb1655bb71e30fd302690cd5ac7937197add94a3ac1cab5812a1f5a48" }, "profile_matrix": [ { diff --git a/crates/tracedecay-query/src/search_quality/packaged.rs b/crates/tracedecay-query/src/search_quality/packaged.rs index 774f01bb55..4b3556a777 100644 --- a/crates/tracedecay-query/src/search_quality/packaged.rs +++ b/crates/tracedecay-query/src/search_quality/packaged.rs @@ -7,7 +7,7 @@ use super::candidate_output::{ use super::evaluate::SearchEvalError; const WORKLOAD_PATH: &str = "tests/fixtures/search_quality/query-lexical-graph-workload-v1.json"; -const WORKLOAD_SHA256: &str = "20322067510f57f5fa75f68674b18290a65d0535af7da363b6b6f29384042ca4"; +const WORKLOAD_SHA256: &str = "267e2bd2e9b90d258cbeed829920ab735f6af0ebc2a6e870d59eeef29b1cdb93"; const FILES: &[(&str, &[u8])] = &[ ( diff --git a/crates/tracedecay-search-eval/src/bin/tracedecay-search-eval-direct.rs b/crates/tracedecay-search-eval/src/bin/tracedecay-search-eval-direct.rs index 1a2b3d692a..eca9235503 100644 --- a/crates/tracedecay-search-eval/src/bin/tracedecay-search-eval-direct.rs +++ b/crates/tracedecay-search-eval/src/bin/tracedecay-search-eval-direct.rs @@ -246,7 +246,7 @@ mod tests { assert_eq!(summary.status, DirectEvaluationStatusV1::Pass); assert_eq!( summary.workload_digest, - "sha256:c7c97a6ab08da36ba02a89ca0d705dee0cc62d3a12d6bd3698f6c2185dd2d708" + "sha256:8657aa486a4c58e17c9969c7aa5d143a4d30e88dca7d26f13e61c7d3effab091" ); assert_eq!(summary.profile_count, 1); assert_eq!(summary.query_count, 67); From 93915560fa4885251dfe02bca90d5fd87fb8a26d Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 18:47:54 +0000 Subject: [PATCH 07/19] style(code-index): allow the held slot across the freshness await CI clippy runs `--all-targets --all-features -- -D warnings` and the crate denies `clippy::all`, so `await_holding_lock` fails the gate on `dashboard_freshness_does_not_join_a_clone_backfill_slice`. Holding the clone-successor slot across the dashboard read is the scenario that test exists to pin: the read must answer without joining the backfill that owns the slot. Carry the same allow and rationale the two scheduler-guard tests in this file already use. Co-Authored-By: Claude Fable 5.1 --- .../src/code_index_scheduler/tests/serving.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs index dd5516dc71..4455c4a5bc 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs @@ -912,6 +912,10 @@ fn clone_status_distinguishes_unavailable_backfill_partial_ready_and_stale() { )); } +// Holding the clone-successor slot across the await is the scenario, not an +// oversight: the read under test must answer without joining the backfill that +// owns the slot. The guard is released before shutdown. +#[allow(clippy::await_holding_lock)] #[tokio::test] async fn dashboard_freshness_does_not_join_a_clone_backfill_slice() { let fixture = GitFixture::new(&[( From c157d22eae940cecf98dd7cf6d4d0808a2676e61 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 18:48:07 +0000 Subject: [PATCH 08/19] test(code-index): expect the typed text-budget refusal 0ae9fad50e made the text-artifact reservation gate keep its typed detail instead of collapsing every refusal into BudgetExceeded, and added a pre-admission check that refuses with AuthorityUnavailable when the headroom below the resident-memory watermark is under the requested minimum. It updated three sibling expectations but not this one, so the clone-successor refusal asserted a variant the production path no longer returns: left: Err(AuthorityUnavailable("text-artifact build needs at least 134217728 bytes; 67108864 bytes are available below the resident-memory watermark")) right: Err(BudgetExceeded) The test still requires a refusal, and still requires the V14 owners to stay queryable and the successor to stay pending; only the variant moves. Co-Authored-By: Claude Fable 5.1 --- .../src/code_index_scheduler/tests/serving.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs index 4455c4a5bc..821d336fad 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs @@ -1165,9 +1165,11 @@ fn transient_clone_successor_reservation_refusal_retries_without_cooling_v14_own ); scheduler.bind_resident_memory(Arc::clone(&resident_memory)); let latest = scheduler.latest_complete().expect("restored generation"); - assert_eq!( - latest.advance_text_serving(1), - Err(tracedecay_query::retrieval::RetrievalPortError::BudgetExceeded), + assert!( + matches!( + latest.advance_text_serving(1), + Err(tracedecay_query::retrieval::RetrievalPortError::AuthorityUnavailable(_)) + ), "the competing reservation must deny the first successor admission" ); latest From 5de202798a67e5e99cc578feca207fc9c051126b Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 18:48:22 +0000 Subject: [PATCH 09/19] test(code-index): select the trait declaration, not its impls 878ae3ee55 gave Rust trait-impl methods the owner `` so two impls of one trait no longer collide. The unresolved-dispatch assertion picked its expected trait method with `qualified_name.contains("Processor")`, which now also matches `::process` and `::process`. The first match by occurrence order was an impl method, so the test demanded that a `resolve_trait_dispatch: false` callee page contain an impl it also asserts must be absent. Select the declaration by its exact owner. The graph already returned the right edge: `src/lib.rs::Processor::process`. Co-Authored-By: Claude Fable 5.1 --- .../src/code_index_scheduler/tests/serving.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs index 821d336fad..93b25ad191 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs @@ -4734,8 +4734,11 @@ async fn callable_application_operations_consume_exact_lexical_and_graph_owners( .symbols .iter() .find(|record| { + // Trait-impl methods are owned by ``, so a + // `contains("Processor")` probe also matches every impl of the + // trait. Only the declaration itself is owned by the trait. record.simple_name == "process" - && record.qualified_name.contains("Processor") + && record.qualified_name.ends_with("::Processor::process") && record.kind == "method" }) .expect("trait method symbol") From 7d10aa311c09862973e3a6122681a90903c2ba9a Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 18:48:46 +0000 Subject: [PATCH 10/19] test(code-index): seat the crafted clone-backfill owner Both clone-backfill tests build the "V14 ready, successor pending" owner on a standalone scheduler and inject it into the mounted registry as `text_generation`, leaving the registry seated on its own generation. The worker only drives a retained text projection when the seated generation is that text owner (`drive_retained = !owners_ready || (serving_matches_text && source_current)` in registry/mount.rs), so an unseated crafted owner is never advanced and the pair asserted states the daemon cannot reach: - expired_source_proof_reschedules_pending_clone_backfill waited out its 10 s ceiling with pending_wake=Some(0) and the successor untouched. - query_admission_serves_v14_while_clone_successor_is_pending compared the seat id to the crafted id, which cannot match: `captured_at` is part of the intake digest, so two captures of one checkout never mint one id. Seat the crafted owner as well. The expired-proof test now settles in 0.8 s instead of timing out, which is the successor pass the test exists to prove. Co-Authored-By: Claude Fable 5.1 --- .../src/code_index_scheduler/tests/serving.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs index 93b25ad191..c6b185ada1 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs @@ -990,6 +990,16 @@ async fn query_admission_serves_v14_while_clone_successor_is_pending() { let worktree = mounted .get(&fixture.path().canonicalize().expect("canonical root")) .expect("mounted worktree"); + // Generation identity binds the capture instant (`captured_at` is in + // the intake digest), so the crafted owner and the registry's own + // capture of the same checkout never share an id. Seat the crafted + // owner too: a text owner that is not the seated generation is a state + // the daemon never produces, and the worker's clone-backfill gate + // (`serving_matches_text`) refuses to drive it. + *worktree + .serving_generation + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(latest.clone()); *worktree .text_generation .write() @@ -1064,6 +1074,13 @@ async fn expired_source_proof_reschedules_pending_clone_backfill() { let worktree = mounted .get(&fixture.path().canonicalize().expect("canonical root")) .expect("mounted worktree"); + // Seat the crafted owner alongside its text handle: the worker's + // clone-backfill gate only drives a text owner that is the seated + // generation, and a daemon never holds one that is not. + *worktree + .serving_generation + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(latest.clone()); *worktree .text_generation .write() From ead0844e656228e94e077dbc8126459057439ac5 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 18:49:10 +0000 Subject: [PATCH 11/19] test(code-index): wait for settled clone-index coverage d86fd22b49 made `clone_index_status` read the clone-successor slot with `try_lock` so a freshness read never joins a running backfill; a busy slot answers `Unavailable { "clone-index status is being updated" }`. That is the right production answer, but the coverage test sampled the dashboard once after the edit and demanded ready: the changed V16 artifact must return to ready: Some(Unavailable { reason: "clone-index status is being updated" }) `wait_for_dashboard_ready` gates on staleness and coverage, neither of which tracks the successor slot, so the read raced the backfill of the republished generation. Poll for the settled status instead. Every coverage and update assertion is unchanged. Co-Authored-By: Claude Fable 5.1 --- .../code_index_scheduler/tests/reconcile.rs | 54 ++++++++++++------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs index 3fcf5b0e15..b86209fa71 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs @@ -669,6 +669,38 @@ async fn registry_feeds_publications_and_bounded_freshness_reads() { assert_ne!(changed.generation_id, initial.generation_id); } +/// Poll a mounted worktree's dashboard clone-index status until it reports +/// ready coverage. +/// +/// `clone_index_status` reads the clone-successor slot with `try_lock` so a +/// freshness read never joins a running backfill. A single sample therefore +/// reports `Unavailable { "clone-index status is being updated" }` whenever a +/// freshly published generation's successor still holds the slot, which is a +/// truthful transient, not the settled answer a caller is asking for. +async fn wait_for_ready_clone_index( + registry: &CodeIndexSchedulerRegistryV1, + path: &Path, +) -> tracedecay_contracts::code_index_freshness::CodeCloneIndexObservationV1 { + let deadline = Instant::now() + SERVING_SEAT_FAILURE_CEILING; + loop { + let status = registry + .dashboard_freshness(path) + .await + .expect("mounted dashboard freshness") + .clone_index; + match status { + Some(tracedecay_contracts::code_index_freshness::CodeCloneIndexStatusV1::Ready { + observation, + }) => return observation, + transient => assert!( + Instant::now() <= deadline, + "the V16 artifact never reported ready clone coverage: {transient:?}" + ), + } + tokio::time::sleep(Duration::from_millis(5)).await; + } +} + #[tokio::test] async fn registry_clone_freshness_reports_coverage_and_update_accounting() { let fixture = GitFixture::new(&[("src/lib.rs", "pub fn alpha() -> u32 { 1 }\n")]); @@ -684,16 +716,7 @@ async fn registry_clone_freshness_reports_coverage_and_update_accounting() { .expect("mount worktree"); let initial = wait_for_initial_generation(®istry, fixture.path()).await; wait_for_dashboard_ready(®istry, fixture.path()).await; - let initial_status = registry - .dashboard_freshness(fixture.path()) - .await - .expect("initial clone freshness"); - let Some(tracedecay_contracts::code_index_freshness::CodeCloneIndexStatusV1::Ready { - observation, - }) = initial_status.clone_index - else { - panic!("a complete V16 artifact must report ready clone coverage"); - }; + let observation = wait_for_ready_clone_index(®istry, fixture.path()).await; assert_eq!(observation.coverage.source_bodies, Some(1)); assert_eq!(observation.coverage.eligible_source_bodies, Some(0)); assert_eq!(observation.coverage.conservative_normalized_bodies, Some(0)); @@ -711,16 +734,7 @@ async fn registry_clone_freshness_reports_coverage_and_update_accounting() { )); let _ = wait_for_generation_change(®istry, fixture.path(), &initial).await; wait_for_dashboard_ready(®istry, fixture.path()).await; - let changed = registry - .dashboard_freshness(fixture.path()) - .await - .expect("changed clone freshness"); - let Some(tracedecay_contracts::code_index_freshness::CodeCloneIndexStatusV1::Ready { - observation, - }) = changed.clone_index - else { - panic!("the changed V16 artifact must return to ready"); - }; + let observation = wait_for_ready_clone_index(®istry, fixture.path()).await; assert_eq!(observation.coverage.payloads_reused, Some(0)); assert_eq!(observation.resources.stale_invalidations, Some(1)); assert!(observation.resources.changed_symbol_update_micros.is_some()); From 6318c180d20056e5813eb0d00f876f39bd0ef667 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 18:51:15 +0000 Subject: [PATCH 12/19] test(code-index): attribute probe receipts by their arrival The suppressed-probe assertion compared a receipt count taken just before the probe with one taken 50 ms after, which charges the probe for any wake that was already in flight. Since the seat stopped waiting for the clone successor the mount leaves pending backfill, and each drain wake posts its own Noop receipt, so the count grew inside the window (left: 3, right: 2) with the extra receipt carrying a wake_micros from before the probe. Settle the mount-era chain first, and attribute receipts by the arrival the pass claimed rather than by list position. The assertion is stricter: a probe-era receipt now fails wherever it lands in the list. Add `pending_wake_micros_for_root` so a test holding only the checkout path can see the pending-wake slot, as the scope variant already allows. Co-Authored-By: Claude Fable 5.1 --- .../registry/test_gates.rs | 19 +++++++ .../code_index_scheduler/tests/reconcile.rs | 54 +++++++++++++++---- 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/test_gates.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/test_gates.rs index 4ad148ff43..102974e294 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/test_gates.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/test_gates.rs @@ -521,6 +521,25 @@ impl CodeIndexSchedulerRegistryV1 { }) } + /// The pending-wake slot for one exact mounted root, in unix micros; `0` + /// means no wake is outstanding. A pass that ends while a wake is already + /// pending re-arms a busy follow-up whose receipt lands later, so a test + /// pinning wake or receipt accounting needs this as well as + /// `reconcile_in_progress_for_test`. + #[cfg(test)] + pub(crate) async fn pending_wake_micros_for_root(&self, project_root: &Path) -> Option { + let project_root = project_root.canonicalize().ok()?; + let mounted = self.mounted.lock().await; + mounted.get(&project_root).map(|worktree| { + worktree + .pending_wake + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .micros + }) + } + /// The exact-source currency witness for one mounted root, so tests can /// stage the unproven-seat state a restart restore leaves behind. #[cfg(test)] diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs index b86209fa71..1723e72d95 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs @@ -669,6 +669,30 @@ async fn registry_feeds_publications_and_bounded_freshness_reads() { assert_ne!(changed.generation_id, initial.generation_id); } +/// Wait until the mounted worker for `path` is idle with nothing queued. +/// +/// [`wait_for_quiescent_owner_pass`] only reports that no pass is *running*. +/// A pass that ends while a wake is already pending re-arms a busy follow-up +/// whose receipt lands later, so a test pinning receipt accounting has to wait +/// for the pending-wake slot as well. +async fn wait_for_settled_owner(registry: &CodeIndexSchedulerRegistryV1, path: &Path) { + let deadline = Instant::now() + SERVING_SEAT_FAILURE_CEILING; + loop { + wait_for_quiescent_owner_pass(registry, path).await; + if registry.pending_wake_micros_for_root(path).await == Some(0) + && !registry.reconcile_in_progress_for_test(path).await + { + return; + } + assert!( + Instant::now() <= deadline, + "the owner for {} never settled", + path.display() + ); + tokio::time::sleep(Duration::from_millis(2)).await; + } +} + /// Poll a mounted worktree's dashboard clone-index status until it reports /// ready coverage. /// @@ -3821,10 +3845,13 @@ async fn unchanged_background_freshness_probe_posts_no_overflow_wake() { .await .expect("mount daemon-owned scheduler"); wait_for_initial_generation(®istry, fixture.path()).await; - // The seat is published mid-pass and the receipt lands after the pass - // releases its in-progress guard, so sample the baseline only once the - // mount's own receipt exists, or it is charged to the probe below. - wait_for_quiescent_owner_pass(®istry, fixture.path()).await; + // The seat no longer waits for the clone successor, so the mount leaves + // pending backfill behind. Draining it is a wake of its own, and every + // wake posts its own receipt, so settle the whole mount-era chain first: + // a pass that ends with a wake still pending re-arms a busy follow-up + // whose receipt would otherwise land inside the probe's window below. + drain_clone_backfill(®istry, fixture.path()).await; + wait_for_settled_owner(®istry, fixture.path()).await; wait_for_event_to_ready(®istry).await; let canonical = fixture.path().canonicalize().expect("canonical fixture"); { @@ -3836,7 +3863,11 @@ async fn unchanged_background_freshness_probe_posts_no_overflow_wake() { .policy .staleness_threshold = Duration::ZERO; } - let receipts_before = registry.event_to_ready_receipts().len(); + // Receipts are attributed by the arrival the pass claimed, not by list + // position: a mount-era wake claimed before this instant belongs to the + // mount even when its receipt lands during the window below. Only a wake + // accepted from here on is the probe's. + let probe_at = tracedecay_contracts::now_micros().0; assert_eq!( registry.probe_freshness_admission(fixture.path()).await, @@ -3855,10 +3886,15 @@ async fn unchanged_background_freshness_probe_posts_no_overflow_wake() { Some(0), "matching Git/stat evidence must not become an overflow hint" ); - assert_eq!( - registry.event_to_ready_receipts().len(), - receipts_before, - "a suppressed probe must not fabricate a reconcile receipt" + let receipts = registry.event_to_ready_receipts(); + assert!( + receipts.iter().all(|receipt| { + receipt + .arrival + .wake_micros() + .is_none_or(|wake_micros| wake_micros < probe_at) + }), + "a suppressed probe must not fabricate a reconcile receipt: {receipts:#?}" ); drop(mounted); registry.shutdown().await; From af7e07cf7919690c2d39c83381c38d14acf9e609 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 19:19:19 +0000 Subject: [PATCH 13/19] fix(code-index): retry a contended code-generation store lock A publication refused because another holder owns the code-generation store lock left the worktree stale until an unrelated wake. The refusal is exactly the class `is_transient_capacity_failure` documents: bounded shared capacity already held, released by its holder without notifying this worktree. It was not classified there, so the worker skipped its self-scheduled retry, and the restored arrival sat in the pending-wake slot with no permit behind it. distinct_stores_reconcile_in_parallel_under_bounded_admission caught it on a loaded runner: the first worktree, released from its held scheduler lock, ran one pass that failed with "the publication authority is unavailable: code-generation store has an active owner", posted no receipt, and never ran again, so the wait timed out at its 2 minute ceiling (observed in CI job 105663734109 at 121.07s). Under `taskset -c 0,1` it reproduced in about one run in five, and passes 25/25 with the classification in place. Both the refusal and the classifier now read one shared detail token so they cannot drift apart. Co-Authored-By: Claude Fable 5.1 --- .../src/code_index_scheduler/publication_store.rs | 11 ++++++++++- .../src/code_index_scheduler/reconcile.rs | 8 ++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs index 9b0226b88a..3422489f5e 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs @@ -143,6 +143,15 @@ impl SharedCodeIndexBytePoolV1 { /// every unpinned query and must not be evictable by cursor traffic over /// superseded generations. pub(super) const DECODED_GENERATION_CACHE_CAPACITY: usize = 4; +/// The exact detail a `try_acquire_code_generation_store_lock` refusal carries. +/// +/// The store lock is a bounded shared resource: a concurrent publication in +/// the same store root holds it and releases it on its own. Both the producer +/// below and +/// [`CodeIndexSchedulerErrorV1::is_transient_capacity_failure`] read this one +/// token, so the retry classification cannot drift from the refusal it names. +pub(super) const CODE_GENERATION_STORE_ACTIVE_OWNER_DETAIL_V1: &str = + "code-generation store has an active owner"; /// Whether one generation resolution may enter the single-flight sealed-decode. /// @@ -2150,7 +2159,7 @@ impl CodeIndexAtomicPublicationPort for DaemonCodeIndexPublicationStoreV1 { }; let _store_lock = try_acquire_code_generation_store_lock(store_root) .map_err(Self::unavailable)? - .ok_or_else(|| Self::unavailable("code-generation store has an active owner"))?; + .ok_or_else(|| Self::unavailable(CODE_GENERATION_STORE_ACTIVE_OWNER_DETAIL_V1))?; let prior_pointer = if let Some(expected) = undecoded_expectation.as_ref() { if expected_active_generation.is_some() { return Err(CodeIndexPublicationStoreErrorV1::CompareAndSwap); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index c7aa4b14a4..c2c7eecdde 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -398,6 +398,14 @@ impl CodeIndexSchedulerErrorV1 { } Self::SnapshotMemoryCapacityUnavailable => true, Self::GraphProjection(CodeGraphProjectionError::BudgetExhausted { .. }) => true, + // The code-generation store lock is bounded shared capacity: a + // concurrent publication in the same store root already holds it, + // and it releases on its own without waking this worktree. Every + // other `Unavailable` detail names a fault in this store, so only + // this one refusal is retried. + Self::Production(CodeIndexProductionErrorV1::Publication( + CodeIndexPublicationStoreErrorV1::Unavailable(detail), + )) => detail == super::publication_store::CODE_GENERATION_STORE_ACTIVE_OWNER_DETAIL_V1, _ => false, } } From c76ede6f045b9b09b931c24a3d757217f297409e Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 18:52:17 +0000 Subject: [PATCH 14/19] fix(global-db): refuse a vanished projection output row `converge_released_output_rendering` supersedes an existing `session_messages` row; it never inserts one. #1775 (c55058a3ac) began recording every `OutputCollision` for convergence and #1781 (b7c9b0d2b5) added the `row_missing` session arm, but `ProjectionRowsBatch` derives its session keys from the message rows it found, so a deleted message row is reported as a missing *session* row. The audit therefore recorded a repair that wrote nothing, went green, and left the output permanently gone. Record for convergence only when the output row is still there. A stale row is still repaired (#1775) and a missing session row beside a present message row is still inserted (#1781); a vanished output row is the hard failure both commits promised. The update-side test asserted the pre-#1775 refusal, so it now asserts the intended contract: reopen succeeds and re-projects the tampered body. Co-Authored-By: Claude Fable 5.1 --- .../src/schema_contract/invariants/audit.rs | 18 ++++++++++++++-- .../observation_projection/failure_audit.rs | 21 +++++++++++++++---- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs index a4aa317bd3..565c0a309d 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs @@ -815,12 +815,25 @@ async fn validate_message_projection_row( resolved.released, )? == StoredProvenanceRendering::Current { + // Convergence supersedes an existing output row; it never inserts one. + // Both repair arms below therefore require the row to be there: a + // vanished output stays the hard failure #1775 and #1781 both promised, + // instead of a recorded repair that writes nothing. The batch also + // derives its session keys from the message rows it found, so a missing + // message is reported as a missing *session* row, which is why this + // guard has to cover the session arm too. + let owner_message = owner_projection.message(); + let output_row_present = resolved + .projection_rows + .message(&owner_message.provider, &owner_message.message_id) + .is_some(); match verify_owner_output_rows(conn, resolved, &owner_projection).await { Ok(()) => {} Err(ProjectionStoreError::OutputCollision { provider, message_id, - }) if provider == owner_projection.message().provider + }) if output_row_present + && provider == owner_projection.message().provider && message_id == owner_projection.message().message_id => { // Ownership was validated above, the immutable observation @@ -835,7 +848,8 @@ async fn validate_message_projection_row( provider, session_id, field: "row_missing", - }) if provider == owner_projection.session().provider + }) if output_row_present + && provider == owner_projection.session().provider && session_id == owner_projection.session().session_id => { // The uniquely owned current output has no session row. The diff --git a/crates/tracedecay/tests/session_suite/observation_projection/failure_audit.rs b/crates/tracedecay/tests/session_suite/observation_projection/failure_audit.rs index dbe49ae9e7..ea006aff50 100644 --- a/crates/tracedecay/tests/session_suite/observation_projection/failure_audit.rs +++ b/crates/tracedecay/tests/session_suite/observation_projection/failure_audit.rs @@ -767,8 +767,15 @@ async fn authority_reopen_accepts_historical_generation_after_supersession() { ); } +/// A projected message row is derived state, not authority: the immutable +/// observation plus its uniquely owned current provenance re-derive it exactly. +/// Since #1775 (`c55058a3ac`) the reopen audit therefore repairs a diverged +/// output row through the released-rendering convergence ledger instead of +/// degrading the profile forever. Provenance identity, digests that match +/// neither the current nor the stored output, foreign ownership, and +/// conflicting session fields remain hard failures. #[tokio::test] -async fn projected_message_update_invalidates_audit_and_fails_reopen() { +async fn projected_message_update_is_repaired_on_reopen() { let tmp = audited_projection_fixture("session-audit-update", "message-audit-update").await; let runtime = profile_runtime(&tmp).await; let database_path = runtime @@ -786,13 +793,19 @@ async fn projected_message_update_invalidates_audit_and_fails_reopen() { .unwrap(); drop(raw_conn); + let reopened = HostAdmissionTestRuntimeV1::profile(tmp.path().join(".tracedecay")) + .await + .expect("a diverged output row must be repaired, not refused"); + drop(reopened); assert!( - HostAdmissionTestRuntimeV1::profile(tmp.path().join(".tracedecay")) - .await - .is_err() + projected_message_texts(&tmp).await[0].contains("audited projection body"), + "reopen accepted the tampered body instead of re-projecting it" ); } +/// The repair above covers a diverged row, never a vanished one: nothing in the +/// convergence ledger inserts a missing message row, so a store whose projected +/// output disappeared still has to be named rather than silently admitted. #[tokio::test] async fn projected_message_delete_invalidates_audit_and_fails_reopen() { let tmp = audited_projection_fixture("session-audit-delete", "message-audit-delete").await; From 324d8a7f582f8ca587922af1779153ed2b9217fc Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 19:04:34 +0000 Subject: [PATCH 15/19] test(work): give the attempt-read journey its own concurrency `work_attempt_consumers_read_the_public_start_attempt_effect` leaves the second attempt in flight and then reads it. The Work read handlers are synchronous: `load_effect_dispatch` holds its tokio worker thread while waiting for the exact-SQL transaction slot the in-flight attempt worker owns. On the fixture's two worker threads the reader and that worker deadlock until the 30s transaction-idle reclaim releases the slot. Measured per tool call: `tracedecay_work_execution_history` took 30.0047s with two worker threads and 12ms with eight, against its own 30s Work deadline contract, so the journey resolved by a millisecond-wide race. CI lost it and reported tool_dispatch_deadline_exceeded; a local 96-core run won it by 4ms. Four worker threads carry the reader and the attempt worker, and the run drops from 33.5s to 3.5s. The underlying blocking-read starvation is unchanged and reported separately. Co-Authored-By: Claude Fable 5.1 --- .../tests/mcp_suite/mcp_handler_test/work_test.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/work_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/work_test.rs index 30ad041222..8a8330919d 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/work_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/work_test.rs @@ -147,7 +147,17 @@ async fn configure_attempt_provider(production: &ProductionCompositionFixture) { /// A fresh provider attempt has no session association yet. The public Work /// reads must still project it from the authority that committed the attempt. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +/// +/// The journey deliberately leaves the second attempt in flight and then reads +/// it, so the runtime has to carry the reader and the attempt worker at once. +/// The Work read handlers are synchronous: `load_effect_dispatch` occupies its +/// worker thread while it waits for the exact-SQL transaction slot the in-flight +/// attempt holds. With two worker threads the reader and the attempt worker +/// deadlock until the 30s transaction-idle reclaim frees the slot, which is +/// within milliseconds of this operation's own 30s deadline, so the journey +/// passed or failed by a race rather than by its contract. Provision the +/// threads the journey's own concurrency needs. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn work_attempt_consumers_read_the_public_start_attempt_effect() { let production = production_composition_fixture().await; let project_root = production.project_root.clone(); @@ -390,7 +400,6 @@ async fn work_attempt_consumers_read_the_public_start_attempt_effect() { second_started["identity"]["attempt_id"], "attempt.mcp-attempt-read.second", "{second_started}" ); - let attempts = call( &server, "tracedecay_work_list_attempts", From 4fd4ee73842235673a6d7dab7f9e5fd5d042c980 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 20:04:46 +0000 Subject: [PATCH 16/19] test(transport): make the refresh journey finishable and readable `background_refresh_and_reopen_report_only_servable_generations` could not finish on a four-core runner for two independent reasons, neither of which was the 90s budget. The search page still crossed the MCP response budget after #1783 shrank it to three results: 18,084 characters, of which the cursor and candidate provenance are most. The answer is then a preview plus a `tracedecay_retrieve` handle, so `wait_for_current_generation` never saw `code_generation` and spun until the budget expired even though status already read current. Reassemble a truncated response through its handle, the way an agent does and the way mcp_suite's own helper already does; the page stays small for the common answer. The 768-file batch indexed 98,304 symbols into 455 million lexical units and 645 MB on disk. On four cores that took ~61s to commit and then pushed the reopen past the composition harness's own 20s publish gate, so a 400s budget still failed: the journey could not complete at that size. 96 files keep the refresh observable across many polls and every open inside its gate. The wait loops also yielded rather than slept, issuing ~290 status calls a second against the worker they were waiting for. Verified on four cores (taskset -c 0-3, perf profile): 92s FAIL before, 31s PASS after, 3/3 there and 3/3 unconstrained. Co-Authored-By: Claude Fable 5.1 --- .../graph_rebuild_status_test.rs | 77 +++++++++++++++++-- 1 file changed, 69 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay/tests/transport_acceptance_suite/graph_rebuild_status_test.rs b/crates/tracedecay/tests/transport_acceptance_suite/graph_rebuild_status_test.rs index 5b02449a74..4b5eda1bee 100644 --- a/crates/tracedecay/tests/transport_acceptance_suite/graph_rebuild_status_test.rs +++ b/crates/tracedecay/tests/transport_acceptance_suite/graph_rebuild_status_test.rs @@ -25,6 +25,14 @@ use tracedecay_mcp::JsonRpcResponse; const RECEIPT_TIMEOUT: Duration = Duration::from_secs(90); +/// Pause between status polls while the daemon reconciles. +/// +/// Yielding instead spun the awaiting task against the very worker it waits +/// for: the loop issued roughly 290 `tracedecay_status` calls a second, and on +/// a four-core runner that is a whole core spent recomputing freshness rather +/// than sealing the generation under it. +const POLL_INTERVAL: Duration = Duration::from_millis(25); + fn git(project: &Path, args: &[&str]) { let output = Command::new("git") .args(["-c", "core.hooksPath=.git/no-hooks"]) @@ -87,12 +95,52 @@ async fn tool( name: &str, arguments: Value, ) -> Value { - tool_payload( + let payload = tool_payload( &harness .call_tool(project, name, arguments) .await .unwrap_or_else(|error| panic!("{name} failed: {error}")), - ) + ); + let Some(handle) = payload["truncated"] + .as_bool() + .unwrap_or(false) + .then(|| payload["handle"].as_str()) + .flatten() + else { + return payload; + }; + // A response over the budget answers with a preview plus a retrieve + // handle, not with the payload. A generation-scale search page crosses + // that budget on its cursor and candidate provenance alone, so shrinking + // the page cannot keep it under; reassemble the stored response exactly as + // an agent does before reading the top-level fields. + let mut content = String::new(); + let mut offset = 0_u64; + loop { + let page = tool_payload( + &harness + .call_tool( + project, + "tracedecay_retrieve", + json!({ "handle": handle, "offset": offset, "format": "json" }), + ) + .await + .unwrap_or_else(|error| panic!("tracedecay_retrieve failed: {error}")), + ); + content.push_str( + page["content"] + .as_str() + .unwrap_or_else(|| panic!("retrieved page without content: {page}")), + ); + if page["has_more"] != Value::Bool(true) { + break; + } + offset = page["next_offset"] + .as_u64() + .expect("retrieved page next_offset"); + } + serde_json::from_str(&content) + .unwrap_or_else(|error| panic!("{name} retrieved invalid JSON: {error}; text={content}")) } async fn status(harness: &ProductionProjectCompositionHarnessV1, project: &Path) -> Value { @@ -116,9 +164,10 @@ async fn search( project: &Path, query: &str, ) -> Value { - // Keep the page tiny: a generation-scale refresh batch otherwise returns - // multi-dozen-KiB candidate bodies that MCP truncates into a handle, and - // the wait helpers never see top-level `results` / `code_generation`. + // Keep the page tiny so the common answer fits the response budget; a + // page that still crosses it is reassembled through its retrieve handle in + // `tool`, so the wait helpers always see top-level `results` and + // `code_generation`. tool( harness, project, @@ -168,7 +217,7 @@ async fn wait_for_current_generation( return current_generation; } } - tokio::task::yield_now().await; + tokio::time::sleep(POLL_INTERVAL).await; } }) .await @@ -219,17 +268,29 @@ async fn wait_for_background_refresh( } return; } - tokio::task::yield_now().await; + tokio::time::sleep(POLL_INTERVAL).await; } }) .await .unwrap_or_else(|_| panic!("reopen omitted background-refresh status: {last_status}")); } +/// Files in the batch whose arrival the background refresh has to work through. +/// +/// The batch only has to keep one refresh observable across a few status polls. +/// At 768 files it instead indexed 98,304 symbols into 455 million lexical +/// units and 645 MB on disk, which on a four-core runner takes ~61s to commit +/// and then pushes the reopen past the composition harness's own 20s publish +/// gate: no `RECEIPT_TIMEOUT` can rescue that, the journey simply cannot finish. +/// 96 files still take seconds, so `partial_refresh_in_progress` is sampled +/// many times over at [`POLL_INTERVAL`], and every later open stays inside its +/// gate. +const REFRESH_BATCH_FILES: u32 = 96; + fn install_background_batch(isolation_root: &Path, project: &Path) { let staging = isolation_root.join("refresh-batch-staging"); fs::create_dir_all(&staging).expect("background batch staging directory"); - for file_index in 0..768_u32 { + for file_index in 0..REFRESH_BATCH_FILES { let mut source = String::new(); for symbol_index in 0..128_u32 { writeln!( From fd76a986c2f98b38b894f0405019cf8c1b994593 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 20:46:59 +0000 Subject: [PATCH 17/19] fix(work): clamp TaskSession page size to the mounted budget `WorkEvidenceRetrieveRequestV1::page_size` bounds evidence sources in the Work page and validates up to MAX_WORK_ROOTED_EVIDENCE_SOURCES_V1 (100). The TaskSession adapter passed it through as the per-attempt session hydration page size and refused, permanently and unretryably, whenever it exceeded the mounted authority's retrieval budget. The checked-in core query policy caps max_hydrated_results at 16, so every legal request above 16 hydrated TaskSession as `Unavailable` with no way for a client to learn the ceiling. Clamp the per-attempt page size to the mounted budget instead. The budget is still never exceeded, and a short page already reports partial coverage plus a continuation, so the caller can page the rest. Zero stays a refusal. Co-Authored-By: Claude Fable 5.1 --- .../src/work/work_evidence_retrieval.rs | 83 +++++++++++++++---- 1 file changed, 69 insertions(+), 14 deletions(-) diff --git a/crates/tracedecay-application/src/work/work_evidence_retrieval.rs b/crates/tracedecay-application/src/work/work_evidence_retrieval.rs index 099d5b8cad..71957abfbc 100644 --- a/crates/tracedecay-application/src/work/work_evidence_retrieval.rs +++ b/crates/tracedecay-application/src/work/work_evidence_retrieval.rs @@ -22,9 +22,9 @@ use tracedecay_contracts::{ }; use tracedecay_domain::{ AuthorizationRevision, ComponentRevision, EphemeralSanitizedQueryViewV1, FreshnessVectorDigest, - HydrationStateV1, PrincipalId, QueryNormalizationRevision, RetrievalCursor, RetrievalGrainV1, - RetrievalRequest, RetrievalScope, SanitizerRevision, ScoreDomainId, SingleRootScopeV1, - VectorWatermark, + HydrationStateV1, PrincipalId, QueryNormalizationRevision, RetrievalBudget, RetrievalCursor, + RetrievalGrainV1, RetrievalRequest, RetrievalScope, SanitizerRevision, ScoreDomainId, + SingleRootScopeV1, VectorWatermark, }; use tracedecay_query::retrieval::QueryAuthorityV1; use tracedecay_query::retrieval::evidence_lanes::{ @@ -144,9 +144,10 @@ impl WorkTaskSessionEvidenceRetrievalV1 { fn temporal_query( &self, request: &WorkTaskSessionRequestV1, + page_size: u32, ) -> Result { - let page_size = usize::try_from(request.page_size) - .map_err(|_| WorkEvidenceHydrationErrorV1::Unavailable)?; + let page_size = + usize::try_from(page_size).map_err(|_| WorkEvidenceHydrationErrorV1::Unavailable)?; let context_bytes = WORK_EVIDENCE_CONTEXT_BYTES; let execution_limits = ExecutionLimits { candidate_total_bytes: context_bytes as usize, @@ -227,7 +228,11 @@ impl WorkTaskSessionPortV1 for WorkTaskSessionEvidenceRetrievalV1 { request.source.clone(), ) .map_err(|_| WorkEvidenceHydrationErrorV1::NotFoundOrNotAuthorized)?; - let temporal_query = self.temporal_query(&request)?; + let page_size = task_session_page_size( + request.page_size, + authority.profile().retrieval_budget, + )?; + let temporal_query = self.temporal_query(&request, page_size)?; let retrieval_request = retrieval_request(context, &request, authority.as_ref())?; let query = EphemeralSanitizedQueryViewV1::sanitize( task_session_query_text(&request), @@ -255,7 +260,7 @@ impl WorkTaskSessionPortV1 for WorkTaskSessionEvidenceRetrievalV1 { context, request: &request, reauthorization, - page_size: usize::try_from(request.page_size) + page_size: usize::try_from(page_size) .map_err(|_| WorkEvidenceHydrationErrorV1::Unavailable)?, ranking_cursor, }; @@ -364,17 +369,35 @@ fn map_reauthorization_error( } } +/// The per-attempt TaskSession page size the mounted authority can actually +/// serve. +/// +/// `WorkEvidenceRetrieveRequestV1::page_size` bounds evidence *sources* in the +/// Work page (validated up to `MAX_WORK_ROOTED_EVIDENCE_SOURCES_V1`), which is +/// a different quantity from how many ranked session anchors one attempt may +/// hydrate. Passing it through unclamped made every legal Work request above +/// the mounted profile's hydration budget permanently `Unavailable` instead of +/// a served page plus a continuation, so clamp to the budget here. Zero stays a +/// refusal: no budget can serve it. +fn task_session_page_size( + requested: u32, + budget: RetrievalBudget, +) -> Result { + let page_size = requested + .min(budget.max_hydrated_results) + .min(budget.max_candidates_per_lane); + if page_size == 0 { + return Err(WorkEvidenceHydrationErrorV1::Unavailable); + } + Ok(page_size) +} + fn retrieval_request( context: &RequestContext, request: &WorkTaskSessionRequestV1, authority: &QueryAuthorityV1, ) -> Result { - if request.page_size == 0 - || request.page_size > authority.profile().retrieval_budget.max_hydrated_results - || request.page_size > authority.profile().retrieval_budget.max_candidates_per_lane - { - return Err(WorkEvidenceHydrationErrorV1::Unavailable); - } + task_session_page_size(request.page_size, authority.profile().retrieval_budget)?; Ok(RetrievalRequest { principal: PrincipalId::new(context.actor().as_str()) .map_err(|_| WorkEvidenceHydrationErrorV1::Unavailable)?, @@ -890,7 +913,39 @@ mod unit_tests { use tracedecay_contracts::WorkEvidenceHydrationErrorV1; use tracedecay_contracts::retrieval::SessionRetrievalStructuralRefusalV1; - use super::{budget_hydration_refusal, cursor_manifest_hydration_refusal}; + use tracedecay_domain::RetrievalBudget; + + use super::{ + budget_hydration_refusal, cursor_manifest_hydration_refusal, task_session_page_size, + }; + + const fn budget(max_candidates_per_lane: u32, max_hydrated_results: u32) -> RetrievalBudget { + RetrievalBudget { + max_candidates_per_lane, + max_fused_candidates: 32, + max_hydrated_results, + max_hydration_bytes: 65_536, + deadline_micros: None, + } + } + + #[test] + fn task_session_page_size_clamps_to_the_mounted_budget() { + // The checked-in core query fallback policy. A legal Work evidence + // request (up to MAX_WORK_ROOTED_EVIDENCE_SOURCES_V1) must be served, + // not refused, when it asks for more than one attempt can hydrate. + assert_eq!(task_session_page_size(100, budget(32, 16)), Ok(16)); + assert_eq!(task_session_page_size(8, budget(32, 16)), Ok(8)); + assert_eq!(task_session_page_size(100, budget(4, 16)), Ok(4)); + assert_eq!( + task_session_page_size(100, budget(0, 16)), + Err(WorkEvidenceHydrationErrorV1::Unavailable) + ); + assert_eq!( + task_session_page_size(0, budget(32, 16)), + Err(WorkEvidenceHydrationErrorV1::Unavailable) + ); + } #[test] fn task_session_structural_refusals_retain_exact_hydration_causes() { From 7855fbacce47a1ad23c0b8579ea08b55e333069d Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 20:47:10 +0000 Subject: [PATCH 18/19] test(daemon): gate the fan-out TaskSession tail on its authority Until 8e7952f9 ("retire dense FastEmbed path for lexical/graph") this journey returned early unless TRACEDECAY_DISTRIBUTION_FASTEMBED_FIXTURE named an installed distribution package, because only the evaluated federated profile that fixture activated could rank and hydrate TaskSession anchors. That commit deleted the accepted-profile federated authority, left project open mounting the checked-in core exact/lexical/graph policy (a Fallback-mode QueryAuthorityV1), and removed the fixture gate in the same change. QueryAuthorityV1::task_session_score_domain serves only a Federated authority, and the only remaining federated constructor in the tree is a test helper, so the tail this commit added (wait_for_task_session_available) waited out its full 180s deadline on every run and the journey has been unconditionally red on master since. Restore the gate at the same boundary instead of asserting a lane no mounted authority can serve: probe once, assert the typed `task_session` `unavailable` omission that work_route_exposure_conformance already pins in assert_task_session_unavailable, and skip the evidence tail. Everything before it (fan-out, recovery, synthesis, physical restart, byte-exact receipt preservation) keeps running, and the tail runs unchanged as soon as a federated authority is mounted again. Co-Authored-By: Claude Fable 5.1 --- .../advanced_workflow_journey/task_session.rs | 81 ++++++++++++------- .../advanced_workflow_journey_test.rs | 15 +++- 2 files changed, 65 insertions(+), 31 deletions(-) diff --git a/crates/tracedecay/tests/daemon_suite/advanced_workflow_journey/task_session.rs b/crates/tracedecay/tests/daemon_suite/advanced_workflow_journey/task_session.rs index 2eeb7c2387..1ee295b114 100644 --- a/crates/tracedecay/tests/daemon_suite/advanced_workflow_journey/task_session.rs +++ b/crates/tracedecay/tests/daemon_suite/advanced_workflow_journey/task_session.rs @@ -308,7 +308,7 @@ pub(super) fn restart_and_wait_for_task_session( "a second physical restart must preserve the receipt exactly" ); wait_for_code_generation(home, project); - wait_for_task_session_available(&restarted_client, scope); + let _ = task_session_lane_is_mounted(&restarted_client, scope); (restarted_daemon, restarted_client) } @@ -360,32 +360,6 @@ fn code_generation_wait_diagnostics(home: &Path, project: &Path) -> String { } } -/// The core query authority mounts after the first sealed generation is -/// seated, on a deferred owner. Poll the typed SDK until TaskSession evidence -/// hydrates rather than asserting on the mount's timing. -fn wait_for_task_session_available(client: &Client, scope: &TaskSessionEvidenceScope<'_>) { - let deadline = Instant::now() + Duration::from_secs(180); - loop { - let (_, evidence, omissions) = retrieve( - client, - scope.selection, - scope.task_id, - scope.verified_version, - scope.identity, - TemporalModeV1::Current, - ) - .unwrap_or_else(|error| panic!("typed SDK retrieval failed while waiting: {error}")); - if evidence.is_some() { - return; - } - assert!( - Instant::now() < deadline, - "timed out waiting for the mounted query authority to serve TaskSession: {omissions:?}" - ); - std::thread::sleep(Duration::from_millis(250)); - } -} - fn read_active_code_generation( home: &Path, project: &Path, @@ -452,7 +426,14 @@ pub(super) fn assert_available_over_sdk_mcp_and_dashboard( client: &Client, dashboard: &DashboardProcess, scope: TaskSessionEvidenceScope<'_>, -) -> WorkTaskSessionEvidenceV1 { +) -> Option { + if !task_session_lane_is_mounted(client, &scope) { + eprintln!( + "skipping the mounted fan-out TaskSession evidence section; no evaluated federated \ + query authority is mounted for this project" + ); + return None; + } let TaskSessionEvidenceScope { selection, task_id, @@ -718,7 +699,49 @@ pub(super) fn assert_available_over_sdk_mcp_and_dashboard( revoked["value"]["problem"]["retryable"], true, "rank-final participant revocation must tell the dashboard to restart its read: {revoked}" ); - current + Some(current) +} + +/// Whether this project's mounted query authority can serve the TaskSession +/// retrieval lane at all. +/// +/// Before `8e7952f9` ("retire dense FastEmbed path for lexical/graph") this +/// journey skipped unless the caller had installed the byte-pinned FastEmbed +/// distribution package, because only the evaluated federated profile it +/// activated could rank and hydrate TaskSession anchors. That commit deleted +/// the accepted-profile federated authority and left project open mounting the +/// checked-in core exact/lexical/graph policy, a `Fallback`-mode +/// `QueryAuthorityV1`; `task_session_score_domain` serves only a `Federated` +/// one. The production answer is therefore the typed `task_session` +/// `Unavailable` omission, the same contract `work_route_exposure_conformance` +/// pins in `assert_task_session_unavailable`. Keep the capability gate that +/// commit dropped: assert the typed answer, and run the hydration section only +/// when an authority that can serve the lane is mounted. +fn task_session_lane_is_mounted(client: &Client, scope: &TaskSessionEvidenceScope<'_>) -> bool { + let (receipt, evidence, omissions) = retrieve( + client, + scope.selection, + scope.task_id, + scope.verified_version, + scope.identity, + TemporalModeV1::Current, + ) + .unwrap_or_else(|error| panic!("typed SDK TaskSession capability probe failed: {error}")); + assert!( + receipt.is_some(), + "the mounted route must serve the attempt receipt: {omissions:?}" + ); + if evidence.is_some() { + return true; + } + assert!( + omissions.iter().any(|omission| { + omission.relation == "task_session" + && omission.reason == WorkEvidenceOmissionReasonV1::Unavailable + }), + "an unserved TaskSession lane must stay a typed unavailable omission: {omissions:?}" + ); + false } fn assert_available( diff --git a/crates/tracedecay/tests/daemon_suite/advanced_workflow_journey_test.rs b/crates/tracedecay/tests/daemon_suite/advanced_workflow_journey_test.rs index dc364a219e..d4bc01e025 100644 --- a/crates/tracedecay/tests/daemon_suite/advanced_workflow_journey_test.rs +++ b/crates/tracedecay/tests/daemon_suite/advanced_workflow_journey_test.rs @@ -1402,13 +1402,24 @@ fn mounted_fan_out_recovers_then_synthesizes_and_hands_off() { &sealed_receipt, ); let dashboard = task_session::DashboardProcess::start(&home, &project); - let _task_session = task_session::assert_available_over_sdk_mcp_and_dashboard( + // Until `8e7952f9` ("retire dense FastEmbed path for lexical/graph") this + // journey returned early unless the caller had installed the byte-pinned + // FastEmbed distribution package, because only the evaluated federated + // profile that fixture activated could serve the TaskSession retrieval + // lane that the rest of this journey reads. That commit deleted the + // accepted-profile federated authority and the fixture gate together, so + // the tail below has no mounted authority to read. Keep the gate at the + // same boundary: everything above still runs, and the evidence tail runs + // once a federated authority is mounted again. + let Some(_task_session) = task_session::assert_available_over_sdk_mcp_and_dashboard( &home, &project, &client, &dashboard, evidence_scope, - ); + ) else { + return; + }; let (proximity_status, proximity) = dashboard.read_proximity(now()); assert_eq!( proximity_status, 200, From 1828d6e1afef7d0744ebe1fa1045b664f40bd81c Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 21:34:18 +0000 Subject: [PATCH 19/19] test(code-index): attribute the unchanged reconcile by its arrival unchanged_reconcile_does_not_reactivate_the_serving_generation sampled a receipt count right after the initial-generation wait and then asserted that the receipt at that index was the overflow's Noop. The seat is published mid-pass, so the mount's own receipt lands after the pass releases its in-progress guard: on a loaded runner the baseline was taken at 0 and index 0 was the mount's Published receipt, not the overflow's (dumped receipts under `taskset -c 0,1`: [Mount/Published, Overflow/Noop, BusyFollowUp/Noop]). Both nextest tries failed that way in CI run 35390928038, and it reproduced 6 times in 25 runs locally. The list-position assumption is the stale part, exactly as in 6318c180d2. The mount also leaves clone backfill behind now, and each drain wake posts its own receipt, so settle the whole mount-era chain first, then attribute the receipt by the arrival the pass claimed. Settling also matters for correctness of the attribution: `note_wake` keeps the earliest pending instant, so a mount-era wake still pending when the overflow arrives would hand the overflow pass a pre-overflow arrival. Read the serving generation after that settle, since it is the generation the reconcile must retain. `wait_for_settled_owner` moves to the shared test module so both tests use one helper. Passes 30/30 under `taskset -c 0,1`; the probe and parallel-admission siblings pass 10/10 and 6/6. Co-Authored-By: Claude Fable 5.1 --- .../src/code_index_scheduler/tests/mod.rs | 24 ++++++++++++++ .../tests/noop_reconcile_tests.rs | 33 ++++++++++++++++--- .../code_index_scheduler/tests/reconcile.rs | 26 +-------------- 3 files changed, 54 insertions(+), 29 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs index 53ba915841..3b6ba3ceb6 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs @@ -1203,6 +1203,30 @@ async fn wait_for_quiescent_owner_pass( } } +/// Wait until the mounted worker for `path` is idle with nothing queued. +/// +/// [`wait_for_quiescent_owner_pass`] only reports that no pass is *running*. +/// A pass that ends while a wake is already pending re-arms a busy follow-up +/// whose receipt lands later, so a test pinning receipt accounting has to wait +/// for the pending-wake slot as well. +async fn wait_for_settled_owner(registry: &CodeIndexSchedulerRegistryV1, path: &Path) { + let deadline = Instant::now() + SERVING_SEAT_FAILURE_CEILING; + loop { + wait_for_quiescent_owner_pass(registry, path).await; + if registry.pending_wake_micros_for_root(path).await == Some(0) + && !registry.reconcile_in_progress_for_test(path).await + { + return; + } + assert!( + Instant::now() <= deadline, + "the owner for {} never settled", + path.display() + ); + tokio::time::sleep(Duration::from_millis(2)).await; + } +} + /// Drive the seated owner's clone-fingerprint backfill to completion. /// /// The seat no longer waits for that successor: exact and lexical serve as diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/noop_reconcile_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/noop_reconcile_tests.rs index 93ae1042ef..7c1c03a2b0 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/noop_reconcile_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/noop_reconcile_tests.rs @@ -13,8 +13,21 @@ async fn unchanged_reconcile_does_not_reactivate_the_serving_generation() { ) .await .expect("mount"); - let serving_generation = wait_for_initial_generation(®istry, fixture.path()).await; + wait_for_initial_generation(®istry, fixture.path()).await; + // The seat is published mid-pass and the seat no longer waits for the + // clone successor, so the mount's own receipt lands after the in-progress + // guard drops and the leftover backfill drains on later wakes that post + // receipts of their own. Settle that whole chain first: a wake still + // pending when the overflow arrives keeps its earlier arrival instant, and + // the pass would then answer for both. + drain_clone_backfill(®istry, fixture.path()).await; + wait_for_settled_owner(®istry, fixture.path()).await; + wait_for_event_to_ready(®istry).await; let admission = quiesced_background_reconcile_admission(®istry, fixture.path()).await; + let serving_generation = registry + .latest_generation_id(fixture.path()) + .await + .expect("serving generation"); let scheduler = registry .scheduler_handle(fixture.path()) .await @@ -29,7 +42,11 @@ async fn unchanged_reconcile_does_not_reactivate_the_serving_generation() { .worktree .clone() .expect("worktree identity"); - let before_receipts = registry.event_to_ready_receipts().len(); + // Receipts are attributed by the arrival the pass claimed, not by list + // position: a mount-era receipt that lands after this instant still + // belongs to the mount. Only a wake accepted from here on is this + // reconcile's. + let overflow_at = tracedecay_contracts::now_micros().0; // Any redundant graph activation now fails. An unchanged reconcile must // still reach its Noop receipt by retaining the already-serving graph. @@ -45,8 +62,16 @@ async fn unchanged_reconcile_does_not_reactivate_the_serving_generation() { let deadline = std::time::Instant::now() + Duration::from_secs(3); loop { let receipts = registry.event_to_ready_receipts(); - if let Some(receipt) = receipts.get(before_receipts) { - assert!(receipt.is_noop(), "unchanged reconcile must be a no-op"); + if let Some(receipt) = receipts.iter().find(|receipt| { + receipt + .arrival + .wake_micros() + .is_some_and(|wake_micros| wake_micros >= overflow_at) + }) { + assert!( + receipt.is_noop(), + "unchanged reconcile must be a no-op: {receipts:#?}" + ); break; } assert!( diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs index 1723e72d95..41254144b5 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs @@ -34,7 +34,7 @@ use super::{ wait_for_initial_generation, wait_for_live_complete_generation, wait_for_live_complete_generation_by_polling, wait_for_queryable_text_generation, wait_for_queryable_text_generation_change, wait_for_queryable_text_generation_id, - wait_for_quiescent_owner_pass, wait_until_serving_seat, write, + wait_for_quiescent_owner_pass, wait_for_settled_owner, wait_until_serving_seat, write, }; use crate::{ code_index::{ @@ -669,30 +669,6 @@ async fn registry_feeds_publications_and_bounded_freshness_reads() { assert_ne!(changed.generation_id, initial.generation_id); } -/// Wait until the mounted worker for `path` is idle with nothing queued. -/// -/// [`wait_for_quiescent_owner_pass`] only reports that no pass is *running*. -/// A pass that ends while a wake is already pending re-arms a busy follow-up -/// whose receipt lands later, so a test pinning receipt accounting has to wait -/// for the pending-wake slot as well. -async fn wait_for_settled_owner(registry: &CodeIndexSchedulerRegistryV1, path: &Path) { - let deadline = Instant::now() + SERVING_SEAT_FAILURE_CEILING; - loop { - wait_for_quiescent_owner_pass(registry, path).await; - if registry.pending_wake_micros_for_root(path).await == Some(0) - && !registry.reconcile_in_progress_for_test(path).await - { - return; - } - assert!( - Instant::now() <= deadline, - "the owner for {} never settled", - path.display() - ); - tokio::time::sleep(Duration::from_millis(2)).await; - } -} - /// Poll a mounted worktree's dashboard clone-index status until it reports /// ready coverage. ///