diff --git a/.cargo/config.toml b/.cargo/config.toml index ef2197b4ef..ce65ee3f3a 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -67,11 +67,15 @@ rustflags = ["-C", "split-debuginfo=unpacked"] # # `test-ci` is the hosted acceptance selection: nextest's `ci` policy # (.config/nextest.toml: fail-fast off, one retry that still fails flaky -# results, 8 threads, slow-timeout termination) and only the root fixture -# feature. `.github/workflows/ci.yml` runs this exact alias, and its support -# binaries/archives are built with `--profile perf` so they share artifacts -# with it. Extra flags append: `cargo test-ci --locked -E 'binary(=x)'`. -test-ci = "nextest run --workspace --profile ci --features tracedecay/test-helpers --cargo-profile perf" +# results, 8 threads, slow-timeout termination), the root fixture feature and +# `search-eval`, which is what compiles the evaluator acceptance suites (the +# root crate's dependency on the evaluator is optional so the Linux transport +# partition does not build the eval-only lexical projection; a `--workspace` +# run resolves the evaluator anyway). `.github/workflows/ci.yml` runs this +# exact alias, and its support binaries/archives are built with `--profile +# perf` so they share artifacts with it. Extra flags append: `cargo test-ci +# --locked -E 'binary(=x)'`. +test-ci = "nextest run --workspace --profile ci --features tracedecay/test-helpers,tracedecay/search-eval --cargo-profile perf" # Deliberately broader than CI: every optional feature (test-transport # acceptance suites, hotpath, …) under the same cargo profile, with the # default nextest policy (no retries). diff --git a/.claude/skills/using-hotpath/references/hotpath-0.24.md b/.claude/skills/using-hotpath/references/hotpath-0.24.md index a7cb31195e..c4f57b7e57 100644 --- a/.claude/skills/using-hotpath/references/hotpath-0.24.md +++ b/.claude/skills/using-hotpath/references/hotpath-0.24.md @@ -161,6 +161,8 @@ val_logs {"debug_id": 3} Hotpath 0.24 detail calls accept IDs, not names, and have no per-call `limit`. Retention is controlled globally by `HOTPATH_LOGS_LIMIT`. +Published hotpath 0.24 applies `HOTPATH_FUNCTIONS_LIMIT`, else `HOTPATH_LIMIT`, when the exit report is built. Live `functions_timing` and `functions_alloc` use the builder limit captured at guard start. The shipped `tracedecay` process copies that environment onto the builder before the server starts, so a limit set for the process is what those tools return. Setting the variable after the process is already running does not resize the worker. + Recommended order: 1. `profiler_status`. diff --git a/.codex/skills/using-hotpath/references/hotpath-0.24.md b/.codex/skills/using-hotpath/references/hotpath-0.24.md index a7cb31195e..c4f57b7e57 100644 --- a/.codex/skills/using-hotpath/references/hotpath-0.24.md +++ b/.codex/skills/using-hotpath/references/hotpath-0.24.md @@ -161,6 +161,8 @@ val_logs {"debug_id": 3} Hotpath 0.24 detail calls accept IDs, not names, and have no per-call `limit`. Retention is controlled globally by `HOTPATH_LOGS_LIMIT`. +Published hotpath 0.24 applies `HOTPATH_FUNCTIONS_LIMIT`, else `HOTPATH_LIMIT`, when the exit report is built. Live `functions_timing` and `functions_alloc` use the builder limit captured at guard start. The shipped `tracedecay` process copies that environment onto the builder before the server starts, so a limit set for the process is what those tools return. Setting the variable after the process is already running does not resize the worker. + Recommended order: 1. `profiler_status`. diff --git a/.github/linux-test-partitions.json b/.github/linux-test-partitions.json index 9febd5785a..e832748bfa 100644 --- a/.github/linux-test-partitions.json +++ b/.github/linux-test-partitions.json @@ -68,7 +68,8 @@ "example:tracedecay-host-cli-fixture" ], "features": [ - "tracedecay/test-helpers" + "tracedecay/test-helpers", + "tracedecay/search-eval" ] }, { @@ -193,6 +194,7 @@ "tracedecay-framing", "tracedecay-hooks", "tracedecay-host-integration", + "tracedecay-hotpath-guard", "tracedecay-mcp-catalog", "tracedecay-policy", "tracedecay-privacy", @@ -236,6 +238,7 @@ } ], "not_run": { - "tracedecay-global-db::schema_convergence_hotpath": "requires hotpath-alloc, which no test lane enables; the hotpath workflows own it" + "tracedecay-global-db::schema_convergence_hotpath": "requires hotpath-alloc, which no test lane enables; the hotpath workflows own it", + "tracedecay-hotpath-guard::functions_limit_live": "requires hotpath and hotpath-mcp to start a live MCP server, which no test lane enables; the hotpath graph check compiles it" } } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a399aad93..f1c1d1c777 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -219,24 +219,17 @@ jobs: python3 scripts/test-check-dev-skill-mirrors.py python3 scripts/check-dev-skill-mirrors.py check - - name: Validate pushed commit messages - if: github.event_name == 'push' + # Dispatch admits an integration branch. Judge the commits a merge onto + # the default branch would introduce, with the same linter a push uses. + # A push still uses the before SHA, so published history is not rejudged. + - name: Validate commit messages + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' env: + EVENT_NAME: ${{ github.event_name }} BEFORE_SHA: ${{ github.event.before }} HEAD_SHA: ${{ github.sha }} - run: | - set -euo pipefail - if [ "$BEFORE_SHA" = "0000000000000000000000000000000000000000" ]; then - if git rev-parse "${HEAD_SHA}^" >/dev/null 2>&1; then - base_sha="${HEAD_SHA}^" - else - git show --no-patch --format=%B "$HEAD_SHA" | npm run lint:commit -- - exit 0 - fi - else - base_sha="$BEFORE_SHA" - fi - node scripts/lint-commit-range.mjs --repository "$PWD" "$base_sha" "$HEAD_SHA" + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: scripts/lint-ci-commits.sh macos-test-partition: name: Test macOS ${{ matrix.group }} @@ -431,10 +424,14 @@ jobs: # `--lib` / `--test ` / `--bins` decide what compiles, and a filterset # would compile everything and skip at run time. The three root partitions # share one package selection (`tracedecay`, `tracedecay-cli`, - # `tracedecay-search-eval`) with the root fixture feature, so they resolve - # one identical dependency graph and every cargo invocation inside a job - # (the test build, the executables the suites spawn) is a cache hit against - # it. `scripts/linux-test-partitions.py check` proves, from `cargo + # `tracedecay-search-eval`) with the root fixture feature. Journeys also + # enable `tracedecay/search-eval`, the only link from the root crate to the + # evaluator library. It stays off every other partition: an unconditional + # dependency unifies `tracedecay-query/search-eval` into every test + # target of the package, and the transport suites then compile the eval-only + # lexical projection. Every cargo invocation inside a job (the test build, + # the executables the suites spawn) is a cache hit against that job's + # resolution. `scripts/linux-test-partitions.py check` proves, from `cargo # metadata`, that every test target in the workspace is selected by exactly # one partition or listed under `not_run` with a reason, so a new crate or # suite cannot fall out of the lane silently; `scope-gate` derives the @@ -737,7 +734,12 @@ jobs: shared-key: ci-test-full-windows-msvc-lld cache-on-failure: true - # Acceptance tests execute the ordinary evaluator binaries. Match the + # Acceptance tests execute the ordinary evaluator binaries, and + # `tracedecay/search-eval` is what compiles the suites that check the + # CLI receipt against the library. This lane builds `--workspace`, so + # the evaluator and `tracedecay-query/search-eval` already resolve here + # and the feature adds no compilation; the Linux partitions split the + # package selection, which is why it is per-partition there. Match the # archive's features and perf profile to reuse dependency artifacts; # nextest's archive.include carries the executables to every shard. # `--tests` keeps the binaries in the dev-dependency graph the archive @@ -746,14 +748,14 @@ jobs: # lane). - name: Build workspace binaries and tests for the Windows test lane shell: pwsh - run: cargo build --workspace --bins --tests --locked --profile perf --features tracedecay/test-helpers + run: cargo build --workspace --bins --tests --locked --profile perf --features tracedecay/test-helpers,tracedecay/search-eval # `--workspace`, not `-p tracedecay-cli`: the package selection decides # feature unification, and the narrower one recompiles the code-index # and extraction crates in a second configuration (see the Linux lane). - name: Build Windows host-CLI test fixture shell: pwsh - run: cargo build --workspace --example tracedecay-host-cli-fixture --locked --profile perf --features tracedecay/test-helpers + run: cargo build --workspace --example tracedecay-host-cli-fixture --locked --profile perf --features tracedecay/test-helpers,tracedecay/search-eval # The controlled-workload Hotpath parity helpers are provisioned and # verified by the `hotpath-parity` job. Building them here would put a @@ -764,7 +766,7 @@ jobs: # archive alias, so the arguments are spelled out here once. - name: Build nextest archive shell: pwsh - run: cargo nextest archive --workspace --profile ci --locked --features tracedecay/test-helpers --cargo-profile perf --timings --archive-file "$env:RUNNER_TEMP/nextest-archive.tar.zst" + run: cargo nextest archive --workspace --profile ci --locked --features tracedecay/test-helpers,tracedecay/search-eval --cargo-profile perf --timings --archive-file "$env:RUNNER_TEMP/nextest-archive.tar.zst" - name: Upload Windows archive build timings if: always() diff --git a/CHANGELOG.md b/CHANGELOG.md index cc4448d9bf..7c42d0b1c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1710,6 +1710,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- *(admission)* `NotApplicable` is a terminal no-op in the shared replay-pass + decision. A closed status that leaves the spool unchanged now stops until + the next kick instead of entering the retryable backoff arm. + - *(code-index)* the background worker consults the typed publication-authority park instead of a loop-local bool, so a park it has not yet observed still stops reconcile. Branch publication handles `NotApplicable` as a closed diff --git a/Cargo.lock b/Cargo.lock index 3681964872..b6be9ab4b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5642,6 +5642,7 @@ dependencies = [ "tracedecay-global-db", "tracedecay-hooks", "tracedecay-host-integration", + "tracedecay-hotpath-guard", "tracedecay-lcm", "tracedecay-lsp", "tracedecay-maintenance", @@ -6125,6 +6126,14 @@ dependencies = [ "tracedecay-domain", ] +[[package]] +name = "tracedecay-hotpath-guard" +version = "0.1.0" +dependencies = [ + "hotpath", + "serde_json", +] + [[package]] name = "tracedecay-lcm" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 32920d9169..eb65fc59d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ members = [ "crates/tracedecay-host-admission", "crates/tracedecay-host-integration", "crates/tracedecay-hooks", + "crates/tracedecay-hotpath-guard", "crates/tracedecay-lcm", "crates/tracedecay-lsp", "crates/tracedecay-maintenance", diff --git a/crates/tracedecay-agent-hosts/src/agents/codex.rs b/crates/tracedecay-agent-hosts/src/agents/codex.rs index 6141304f00..d9f9a0246e 100644 --- a/crates/tracedecay-agent-hosts/src/agents/codex.rs +++ b/crates/tracedecay-agent-hosts/src/agents/codex.rs @@ -92,6 +92,17 @@ impl AgentIntegration for CodexIntegration { // Core apply drives `codex plugin add` when the host CLI is present. // When it is not, stop with the same backtick remediation preflight // uses so operators (and lifecycle tests) can activate natively. + // + // `Ready` here is a promise that Core apply can complete, so it must + // not be returned when no `codex` resolves. Returning it anyway opens + // a component transaction that can only die in activation with + // `HostCliUnavailable`; the rollback leaves a `RolledBack` journal + // whose registration backup pins `config.toml` and the versioned + // plugin cache as they were *before* the operator runs the printed + // `codex plugin add`. The next lifecycle command starts with + // `recover_host`, replays that stale rollback over the now-remediated + // host, and refuses with `StalePreview` -- making the remediation this + // very error prints impossible to follow. if plugin_registry::require_codex_plugin_cli().is_err() { let marketplace_name = codex_exact_personal_marketplace_name(&ctx.home) .ok() diff --git a/crates/tracedecay-agent-hosts/src/agents/codex/tests.rs b/crates/tracedecay-agent-hosts/src/agents/codex/tests.rs index 1a6f1b1c9a..bacb3d3374 100644 --- a/crates/tracedecay-agent-hosts/src/agents/codex/tests.rs +++ b/crates/tracedecay-agent-hosts/src/agents/codex/tests.rs @@ -1027,6 +1027,42 @@ fn prepare_stages_the_source_and_returns_ready_for_cli_activation() { ); } +/// `Ready` promises Core apply can drive `codex plugin add`, so an +/// unresolvable plugin CLI must defer instead. +/// +/// Answering `Ready` opens a component transaction that can only die in +/// activation with `HostCliUnavailable`. Its rollback leaves a `RolledBack` +/// journal pinning `config.toml` and the versioned cache as they were before +/// the operator runs the remediation the failure prints, and the next +/// lifecycle command's `recover_host` then refuses the drifted host with +/// `StalePreview`. +#[test] +fn prepare_defers_when_no_plugin_cli_resolves() { + let home = tempfile::tempdir().unwrap(); + // Resolution sees only this empty directory; the process `PATH` (which on + // a developer box usually does carry `codex`) is untouched. + let empty = tempfile::tempdir().unwrap(); + let _host_programs = + tracedecay_runtime_core::config::HostProgramSearchPathGuard::set(empty.path()); + + let outcome = CodexIntegration + .prepare_non_interactive_install(&install_ctx(home.path())) + .unwrap(); + let NonInteractiveInstallOutcome::DeferredUserAction(deferred) = outcome else { + panic!("staging Codex without a resolvable plugin CLI must defer, got {outcome:?}"); + }; + assert!( + deferred + .remediation + .contains("`codex plugin add tracedecay@personal`"), + "the deferral must print the executable remediation: {}", + deferred.remediation + ); + // The source is still staged: the operator's `codex plugin add` consumes it. + assert!(codex_plugin_manifest_path(home.path()).is_file()); + assert!(codex_personal_marketplace_path(home.path()).is_file()); +} + /// Activation must record hook trust even when Codex already reports the /// plugin natively active (no `codex plugin add` run): an already-current /// install can still carry missing or stale trust, and the canonical diff --git a/crates/tracedecay-cli/Cargo.toml b/crates/tracedecay-cli/Cargo.toml index f3dd06146c..ff3248ad49 100644 --- a/crates/tracedecay-cli/Cargo.toml +++ b/crates/tracedecay-cli/Cargo.toml @@ -179,6 +179,7 @@ tracedecay-dashboard-api = { path = "../tracedecay-dashboard-api", version = "0. tracedecay-domain = { path = "../tracedecay-domain", version = "0.1.0" } tracedecay-global-db = { path = "../tracedecay-global-db", version = "0.1.0" } tracedecay-hooks = { path = "../tracedecay-hooks", version = "0.1.0" } +tracedecay-hotpath-guard = { path = "../tracedecay-hotpath-guard", version = "0.1.0" } tracedecay-host-integration = { path = "../tracedecay-host-integration", version = "0.1.0" } tracedecay-lsp = { path = "../tracedecay-lsp", version = "0.1.0" } tracedecay-maintenance = { path = "../tracedecay-maintenance", version = "0.1.0" } diff --git a/crates/tracedecay-cli/src/cli/dispatch.rs b/crates/tracedecay-cli/src/cli/dispatch.rs index 3d81b0779b..7f5fdc4670 100644 --- a/crates/tracedecay-cli/src/cli/dispatch.rs +++ b/crates/tracedecay-cli/src/cli/dispatch.rs @@ -66,13 +66,13 @@ pub async fn resolve_cli_application_surface( execute_application_surface(operation, dispatched, executor).await } -/// Delay before re-sending the same CLI application request when its typed -/// pre-admission problem explicitly directs an after-delay retry. +/// Delay before re-sending the same CLI application request when its completed +/// problem is the publication-window mounting refusal. pub(crate) fn surface_retry_delay(result: &ApplicationSurfaceInvocationResult) -> Option { result .result .as_ref() .err()? .problem - .pre_admission_retry_delay() + .owner_mount_resend_delay() } diff --git a/crates/tracedecay-cli/src/main.rs b/crates/tracedecay-cli/src/main.rs index 647c2851fd..3c5a01c7aa 100644 --- a/crates/tracedecay-cli/src/main.rs +++ b/crates/tracedecay-cli/src/main.rs @@ -523,9 +523,14 @@ fn hotpath_guard() -> hotpath::HotpathGuard { // CPU sampling remains available only by explicit operator request: // `HOTPATH_REPORT` (e.g. `functions-cpu`) takes precedence over this // default exclusion. - hotpath::HotpathGuardBuilder::new("tracedecay") - .sections_exclude(vec![hotpath::Section::FunctionsCpu]) - .build() + // Hotpath 0.24 reads HOTPATH_FUNCTIONS_LIMIT only when the exit report is + // built. Live functions_timing and functions_alloc use the builder limit + // captured when this guard starts, so the same env is applied here. + tracedecay_hotpath_guard::with_functions_display_limit( + hotpath::HotpathGuardBuilder::new("tracedecay") + .sections_exclude(vec![hotpath::Section::FunctionsCpu]), + ) + .build() } #[cfg(feature = "hotpath")] diff --git a/crates/tracedecay-cli/src/tool_command.rs b/crates/tracedecay-cli/src/tool_command.rs index b5f2e0b20f..e9a28e8add 100644 --- a/crates/tracedecay-cli/src/tool_command.rs +++ b/crates/tracedecay-cli/src/tool_command.rs @@ -121,8 +121,6 @@ const PROFILE_REGISTRY_TOOLS: &[&str] = &[ "tracedecay_project_context", ]; -const MAX_SURFACE_ATTEMPTS: usize = 3; - fn tool_deadline_range_error() -> TraceDecayError { TraceDecayError::Config { message: format!( @@ -429,17 +427,13 @@ fn dispatch_cli_application_surface_inner( let handshake = tracedecay::daemon::handshake_for_current_client(project, None, false, false)?; let client = tracedecay_daemon_identity::invocation_client_for_current(handshake)?; - // A cold daemon answers a retryable pre-admission problem while the - // project open still warms in the background (bounded by the daemon's - // foreground open wait). The compatibility tool path rides that state out - // through its project-open retry loop; the typed surface path must present - // the same transport behavior, so re-send the same request per the - // envelope's own retry directive, bounded by both the CLI deadline and - // three attempts so a persistent refusal remains visible to callers. + // A cold daemon answers the mounting refusal while the project open + // still warms in the background. The compatibility tool path rides + // that state out through its project-open retry loop; the typed + // surface path re-sends only that same refusal, until the CLI + // deadline. Every other completed problem is the answer. let mut next_request = Some(request); - let mut attempts = 0usize; let result = loop { - attempts += 1; let request = match next_request.take() { Some(request) => request, None => parse_application_surface_request(operation, tool_args.clone()).map_err( @@ -486,29 +480,18 @@ fn dispatch_cli_application_surface_inner( message: error.to_string(), }, })?; - let Some(delay) = bounded_surface_retry_delay( - crate::cli::dispatch::surface_retry_delay(&result), - attempts, - deadline, - ) else { + let Some(delay) = crate::cli::dispatch::surface_retry_delay(&result) else { break result; }; + if deadline.saturating_duration_since(Instant::now()) <= delay { + break result; + } tokio::time::sleep(delay).await; }; print_cli_application_surface(result, requested_format == RequestedOutputFormat::Json) }) } -fn bounded_surface_retry_delay( - delay: Option, - attempts: usize, - deadline: Instant, -) -> Option { - let delay = delay?; - (attempts < MAX_SURFACE_ATTEMPTS && deadline.saturating_duration_since(Instant::now()) > delay) - .then_some(delay) -} - fn print_cli_application_surface( result: ApplicationSurfaceInvocationResult, raw_json: bool, diff --git a/crates/tracedecay-cli/src/tool_command/tests.rs b/crates/tracedecay-cli/src/tool_command/tests.rs index a685c75973..dd5d9fadde 100644 --- a/crates/tracedecay-cli/src/tool_command/tests.rs +++ b/crates/tracedecay-cli/src/tool_command/tests.rs @@ -92,30 +92,6 @@ fn application_operations_resolve_by_identity_and_by_cli_spelling() { ); } -#[test] -fn retryable_surface_refusals_stop_at_the_attempt_and_deadline_bounds() { - let delay = Duration::from_millis(10); - let roomy_deadline = Instant::now() + Duration::from_secs(1); - assert_eq!( - bounded_surface_retry_delay(Some(delay), 1, roomy_deadline), - Some(delay) - ); - assert_eq!( - bounded_surface_retry_delay(Some(delay), 2, roomy_deadline), - Some(delay) - ); - assert_eq!( - bounded_surface_retry_delay(Some(delay), 3, roomy_deadline), - None, - "the third typed refusal is surfaced instead of retried" - ); - assert_eq!( - bounded_surface_retry_delay(Some(delay), 1, Instant::now() + delay), - None, - "a retry that cannot complete inside the request deadline is refused" - ); -} - #[test] fn whole_payload_invocation_parses_without_a_tool_definition() { let parsed = parse_whole_payload_invocation_with_stdin( diff --git a/crates/tracedecay-cli/src/upgrade.rs b/crates/tracedecay-cli/src/upgrade.rs index 032cf81c0c..da334e24d1 100644 --- a/crates/tracedecay-cli/src/upgrade.rs +++ b/crates/tracedecay-cli/src/upgrade.rs @@ -554,15 +554,15 @@ pub enum UpgradeOutcome { /// binary: `which_tracedecay()`'s current-exe-first order can point /// at the OLD binary (e.g. a stale Homebrew keg) after an upgrade. binary: Option, - /// Version of the freshly installed binary: the release-manifest - /// version for GitHub-release installs, the linked binary's - /// self-reported version for package-manager installs. Daemon restore - /// validates this version, the binary it actually restarts, instead - /// of the one that was running before the upgrade. `None` only when - /// the manager's install could not be interrogated; restore - /// verification then validates the pre-upgrade version and, if a new - /// daemon really was installed, fails with a typed identity mismatch - /// rather than silently passing. + /// Version the installed binary reports for itself (`--version`), + /// `{release}+{sha}[.dirty]`. Daemon restore compares this string to + /// the daemon's advertised build identity exactly, so a release tag + /// is not a substitute: the tag and the binary differ by build + /// metadata, and that mismatch is what failed `tracedecay update`'s + /// readiness wait. `None` only when the binary could not be + /// interrogated; restore then validates the pre-upgrade version and, + /// if a new daemon really was installed, fails with a typed identity + /// mismatch rather than accepting a less specific name. version: Option, }, /// Already on the latest version. The binary was not replaced. @@ -794,11 +794,20 @@ fn run_versioned_upgrade(current: &str, is_beta: bool) -> Result eprintln!("Upgrading v{current} → v{latest}..."); let binary = install_upgrade_version(latest, is_beta)?; record_previous_version(); - eprintln!("\x1b[32m✔\x1b[0m Successfully upgraded to v{latest}!"); - Ok(UpgradeOutcome::Installed { - binary, - version: Some(latest.to_owned()), - }) + Ok(finish_versioned_upgrade(latest, binary)) +} + +/// Completes a GitHub-release install. +/// +/// `catalog_version` is the release name shown to the operator. It is not +/// the installed identity: the published binary names itself +/// `{release}+{sha}` and the daemon advertises that same string. Readiness +/// compares the two exactly, so recording the catalog tag refuses the binary +/// this function just installed. +fn finish_versioned_upgrade(catalog_version: &str, binary: Option) -> UpgradeOutcome { + let version = probed_installed_version(binary.as_deref(), "installed release"); + eprintln!("\x1b[32m✔\x1b[0m Successfully upgraded to v{catalog_version}!"); + UpgradeOutcome::Installed { binary, version } } /// Atomically replaces `target` with the contents of `source`: the bytes are @@ -971,6 +980,25 @@ fn installed_binary_version_within( parse_version_output(&text).ok_or(VersionProbeError::Unrecognized(text)) } +/// The version `binary` reports for itself, or `None` when there is nothing +/// to ask or it does not answer. +/// +/// A missing answer is not filled in from a release tag. The tag omits the +/// commit the binary and the daemon both name, and readiness treats that +/// omission as a different identity. +fn probed_installed_version(binary: Option<&Path>, owner: &str) -> Option { + match installed_binary_version(binary?) { + Ok(version) => Some(version), + Err(reason) => { + eprintln!( + " \x1b[33mwarning:\x1b[0m could not read the {owner} binary's version \ + ({reason}); daemon restore will not invent an identity" + ); + None + } + } +} + /// Whether a delegated manager upgrade was a no-op: the binary the manager /// links reports exactly the build version this process is running, which /// is the same file unless the manager installed something. `None` @@ -1024,17 +1052,8 @@ fn run_delegated_upgrade( None } }; - let installed_version = match binary.as_deref().map(installed_binary_version) { - Some(Ok(version)) => Some(version), - Some(Err(reason)) => { - eprintln!( - " \x1b[33mwarning:\x1b[0m could not read the {label}-installed binary's version \ - ({reason}); assuming a new install so the refresh chain runs" - ); - None - } - None => None, - }; + let installed_version = + probed_installed_version(binary.as_deref(), &format!("{label}-installed")); if delegated_upgrade_was_noop( crate::product_runtime::PRODUCT_BUILD_VERSION, installed_version.as_deref(), @@ -1234,7 +1253,8 @@ mod tests { use tracedecay_runtime_core::git::GitCommandError; use super::super::{ - VersionProbeError, installed_binary_version, installed_binary_version_within, + UpgradeOutcome, VersionProbeError, finish_versioned_upgrade, installed_binary_version, + installed_binary_version_within, }; fn script(dir: &Path, body: &str) -> PathBuf { @@ -1368,6 +1388,47 @@ mod tests { "a successful parent exit does not close an inherited pipe; the deadline must" ); } + + /// The observed update failure: GitHub names the release `0.1.0-beta.47` + /// and the binary that release ships names + /// `0.1.0-beta.47+`. Readiness compares those strings exactly, so + /// the catalog tag must not be the version the outcome records. + #[test] + fn a_release_install_reports_the_binary_identity_not_the_catalog_tag() { + let dir = tempfile::tempdir().unwrap(); + let sha = "84598a0b9c841b914565f46b20bb6c765706e8e5"; + let identity = format!("0.1.0-beta.47+{sha}"); + let binary = script(dir.path(), &format!("printf 'tracedecay {identity}\\n'")); + let catalog = "0.1.0-beta.47"; + + let outcome = finish_versioned_upgrade(catalog, Some(binary)); + + let UpgradeOutcome::Installed { version, .. } = outcome else { + panic!("a published release is an install, got {outcome:?}"); + }; + assert_eq!(version.as_deref(), Some(identity.as_str())); + assert_ne!( + version.as_deref(), + Some(catalog), + "the catalog tag is not the identity the daemon advertises" + ); + } + + /// A binary that cannot be asked must not be labeled with the release + /// tag. Restore then fails closed against the pre-upgrade identity + /// instead of waiting for a version the new daemon will never report. + #[test] + fn an_unreadable_release_binary_is_not_labeled_with_the_catalog_tag() { + let catalog = "0.1.0-beta.47"; + let missing = PathBuf::from("/nonexistent/tracedecay-release"); + + let outcome = finish_versioned_upgrade(catalog, Some(missing)); + + let UpgradeOutcome::Installed { version, .. } = outcome else { + panic!("a published release is an install, got {outcome:?}"); + }; + assert_eq!(version, None); + } } // ── Installation ownership ────────────────────────────────────────── diff --git a/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs index dab98b1f5f..e9b5617f83 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs @@ -15,7 +15,8 @@ use crate::common::{ use serde_json::{Value, json}; use tempfile::TempDir; use tracedecay_contracts::{ - ApplicationProblem, ApplicationProblemEnvelope, RequestId, ResultContractRef, SafeDiagnostic, + ApplicationProblem, ApplicationProblemEnvelope, RUNTIME_MOUNTING_REASON_CODE, RequestId, + ResultContractRef, SafeDiagnostic, }; use tracedecay_domain::UtcMicros; use tracedecay_hooks::{HookEventV2, HookHostV1, HookSpoolConfigV1, HookSpoolV1}; @@ -1937,10 +1938,9 @@ fn spawn_scripted_result_sequence_daemon( } } -/// The MCP tool result the daemon renders for a project route whose retained -/// owner is still mounting behind the core publication: `isError` with the -/// typed pre-admission problem and its after-delay retry directive. -fn mounting_owner_tool_result(retry_after_millis: u64) -> Value { +/// The MCP tool result the daemon renders for a completed pre-admission +/// problem whose retry directive is `after_delay`. +fn retry_directed_tool_result(code: &str, message: &str, retry_after_millis: u64) -> Value { let envelope = ApplicationProblemEnvelope::new( ResultContractRef::new( SchemaId::new("schema.retained.fact_store_add.result").expect("schema id"), @@ -1948,15 +1948,9 @@ fn mounting_owner_tool_result(retry_after_millis: u64) -> Value { ) .expect("result contract"), RequestId::new("request.cli.tool.mounting-owner").expect("request id"), - ApplicationProblem::unavailable( - SafeDiagnostic::new( - "application.surface.unavailable", - "The project runtime for this operation is still mounting", - ) - .expect("diagnostic"), - ), + ApplicationProblem::unavailable(SafeDiagnostic::new(code, message).expect("diagnostic")), ) - .expect("mounting owner envelope") + .expect("retry-directed envelope") .with_retry_after_millis(Some(retry_after_millis)) .expect("retry delay"); json!({ @@ -1969,6 +1963,16 @@ fn mounting_owner_tool_result(retry_after_millis: u64) -> Value { }) } +/// The MCP tool result for a project route whose retained owner is still +/// mounting behind the core publication. +fn mounting_owner_tool_result(retry_after_millis: u64) -> Value { + retry_directed_tool_result( + RUNTIME_MOUNTING_REASON_CODE, + "The project runtime for this operation is still mounting", + retry_after_millis, + ) +} + fn fact_store_add_args() -> String { json!({ "category": "project", @@ -2022,6 +2026,8 @@ fn tool_waits_through_an_after_delay_unavailable_within_its_deadline() { socket_path.clone(), "tracedecay_fact_store_add", vec![ + mounting_owner_tool_result(RETRY_AFTER_MILLIS), + mounting_owner_tool_result(RETRY_AFTER_MILLIS), mounting_owner_tool_result(RETRY_AFTER_MILLIS), mounting_owner_tool_result(RETRY_AFTER_MILLIS), json!({ @@ -2051,25 +2057,24 @@ fn tool_waits_through_an_after_delay_unavailable_within_its_deadline() { "stdout must carry the mounted owner's answer, got:\n{stdout}" ); assert!( - !stdout.contains("application.surface.unavailable"), + !stdout.contains(RUNTIME_MOUNTING_REASON_CODE), "a ridden-out mounting state must not reach the caller, got:\n{stdout}" ); let attempts = std::iter::from_fn(|| daemon.requests.try_recv().ok()).count(); assert_eq!( - attempts, 3, - "the CLI must re-send the same request until the owner answers" + attempts, 5, + "the CLI must re-send the same mounting request until the owner answers" ); assert!( - elapsed >= Duration::from_millis(2 * RETRY_AFTER_MILLIS), + elapsed >= Duration::from_millis(4 * RETRY_AFTER_MILLIS), "each retry must wait the delay the directive names, took {elapsed:?}" ); } -/// A completed typed unavailable is still an answer. After three identical -/// results the CLI returns it even when the caller's deadline is much wider; -/// otherwise a permanent diagnostic reconnects every 250 ms until 120 s. +/// A completed authority unavailable is the daemon's answer. Its `after_delay` +/// directive is for the caller; the CLI must not reconnect on it. #[test] -fn tool_caps_repeated_after_delay_results_before_the_deadline() { +fn tool_returns_a_completed_authority_result_without_resending() { let home = TempDir::new().unwrap(); let project = TempDir::new().unwrap(); let socket_dir = TempDir::new().unwrap(); @@ -2081,7 +2086,11 @@ fn tool_caps_repeated_after_delay_results_before_the_deadline() { let daemon = spawn_scripted_result_sequence_daemon( socket_path.clone(), "tracedecay_fact_store_add", - vec![mounting_owner_tool_result(250)], + vec![retry_directed_tool_result( + "application.retained.authority-unavailable", + "The retained operation authority is unavailable: history is not available", + 250, + )], ); let started = Instant::now(); let output = run_command_with_timeout( @@ -2092,7 +2101,7 @@ fn tool_caps_repeated_after_delay_results_before_the_deadline() { assert!( !output.status.success(), - "an owner that never mounts within the deadline must fail\nstdout:\n{}\nstderr:\n{}", + "a completed authority unavailable must fail typed\nstdout:\n{}\nstderr:\n{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); @@ -2102,8 +2111,8 @@ fn tool_caps_repeated_after_delay_results_before_the_deadline() { }); assert_eq!(printed["isError"], true); assert_eq!( - printed["problem"]["code"], "application.surface.unavailable", - "the daemon's typed state must be surfaced after the result retry cap" + printed["problem"]["code"], "application.retained.authority-unavailable", + "the daemon's completed answer must be surfaced, got:\n{stdout}" ); assert_eq!(printed["problem"]["retry"], "after_delay"); let stderr = String::from_utf8_lossy(&output.stderr); @@ -2112,10 +2121,13 @@ fn tool_caps_repeated_after_delay_results_before_the_deadline() { "the process must fail typed, got:\n{stderr}" ); let attempts = std::iter::from_fn(|| daemon.requests.try_recv().ok()).count(); - assert_eq!(attempts, 3, "completed results have one shared attempt cap"); + assert_eq!( + attempts, 1, + "a completed authority result must not be resent" + ); assert!( - elapsed >= Duration::from_millis(500) && elapsed < Duration::from_secs(5), - "the CLI must honor two retry delays but return well before the 10s deadline, took {elapsed:?}" + elapsed < Duration::from_secs(5), + "the CLI must return the completed answer well before the 10s deadline, took {elapsed:?}" ); } diff --git a/crates/tracedecay-code-extraction/src/rust_extractor.rs b/crates/tracedecay-code-extraction/src/rust_extractor.rs index dc6537912e..c00f740eda 100644 --- a/crates/tracedecay-code-extraction/src/rust_extractor.rs +++ b/crates/tracedecay-code-extraction/src/rust_extractor.rs @@ -28,8 +28,8 @@ struct ShadowedCallNames { } /// Receiver bindings whose type the function body states outright: typed -/// parameters, typed `let`s, and `let`s initialised by a struct literal -/// (`T { .. }`, possibly behind `?`). A dotted +/// parameters, typed `let`s, `let`s initialised by a struct literal +/// (`T { .. }`, possibly behind `?`), and `self` in a method. A dotted /// call on such a binding also names the method by its type /// (`builder.build()` → `ignore::WalkBuilder::build`), which is the only form /// the resolver can bind across files. Method calls and constructor-like names @@ -1579,24 +1579,11 @@ impl RustExtractor { column: child.start_position().column as u32, file_path: state.file_path.clone(), }); - // For dot-calls (e.g. `instance.method()`), also emit - // a ref with just the method name so the resolver can - // match it against impl method definitions. - if let Some(method_name) = callee_name.rsplit('.').next() - && method_name != callee_name - { - state.unresolved_refs.push(UnresolvedRef { - from_node_id: fn_node_id.to_string(), - reference_name: method_name.to_string(), - reference_kind: EdgeKind::Calls, - line: child.start_position().row as u32, - column: child.start_position().column as u32, - file_path: state.file_path.clone(), - }); - } - // A dotted call on a binding with a stated type also - // names the method through its type, the only form - // that binds across files. + // The simple name of a dotted call is not itself a call. + // `items.push()` must not bind a same-file `fn push`. + // Only a stated receiver type names the method + // (`Rows::len`), which is also the form that binds + // across files. if let Some(typed_method) = Self::typed_receiver_method(state, callee, receivers) { @@ -1664,13 +1651,85 @@ impl RustExtractor { } let value = callee.child_by_field_name("value")?; let field = callee.child_by_field_name("field")?; - if value.kind() != "identifier" || field.kind() != "field_identifier" { + if field.kind() != "field_identifier" { return None; } - let type_path = receivers.type_of(state.node_text(value))?; + // `self` is its own token, not an identifier. Both name a binding. + let receiver_name = match value.kind() { + "identifier" | "self" => state.node_text(value), + _ => return None, + }; + let type_path = receivers.type_of(receiver_name)?; Some(format!("{type_path}::{}", state.node_text(field))) } + /// The type `self` names in the enclosing impl or trait, carrying the + /// enclosing module path. + /// + /// Trait impls store `` so the method keeps a UFCS name. + /// `self` still names `Type`, the path a call site writes and the alias + /// same-file resolution binds. Same-file resolution keys a definition by + /// its file-relative qualified name, so an impl inside `mod inner` has to + /// name `inner::Type::method` or the call binds nothing. + fn enclosing_receiver_type(state: &ExtractionState<'_>) -> Option { + let owner = state + .node_stack + .iter() + .rposition(|(_, id)| id.starts_with("impl:") || id.starts_with("trait:"))?; + let (name, id) = &state.node_stack[owner]; + let type_name = if id.starts_with("impl:") { + Self::impl_owner_type_name(name) + } else { + name.as_str() + }; + if type_name.is_empty() + || type_name == "Self" + || type_name == "" + || type_name == "" + { + return None; + } + // Frame 0 is the file root, which the qualified name drops. + let mut path = state + .node_stack + .get(1..owner) + .unwrap_or_default() + .iter() + .map(|(segment, _)| segment.as_str()) + .collect::>(); + path.push(type_name); + Some(path.join("::")) + } + + /// The self type inside a stored impl owner name. + /// + /// A trait impl stores ``, and `Type` can itself be a + /// projection (`::Item`), so the delimiter is the ` as ` at + /// depth zero inside the wrapper, not the first one in the string. + fn impl_owner_type_name(owner: &str) -> &str { + let Some(inner) = owner.strip_prefix('<') else { + return owner; + }; + let mut depth = 0_i32; + for (index, character) in inner.char_indices() { + match character { + '<' => depth += 1, + '>' => { + if depth == 0 { + break; + } + depth -= 1; + } + _ => { + if depth == 0 && inner[index..].starts_with(" as ") { + return inner[..index].trim(); + } + } + } + } + owner + } + /// Records every binding the function introduces with the type it states, /// or `None` for a binding whose type the syntax does not state (pattern /// destructuring, `if let`, `match` arms, closure parameters, `for`). @@ -1681,6 +1740,11 @@ impl RustExtractor { receivers: &mut ReceiverTypes, ) { match node.kind() { + "self_parameter" => { + if let Some(type_path) = Self::enclosing_receiver_type(state) { + receivers.record("self".to_owned(), Some(type_path)); + } + } "parameter" => { if let Some(pattern) = node.child_by_field_name("pattern") { let type_path = node @@ -1737,15 +1801,15 @@ impl RustExtractor { } } - /// A bare identifier pattern takes `type_path`; every identifier inside any - /// other pattern is bound with an unknown type. + /// A bare identifier or `self` pattern takes `type_path`; every identifier + /// inside any other pattern is bound with an unknown type. fn record_receiver_pattern( state: &ExtractionState<'_>, pattern: TsNode<'_>, type_path: Option, receivers: &mut ReceiverTypes, ) { - if pattern.kind() == "identifier" { + if pattern.kind() == "identifier" || pattern.kind() == "self" { receivers.record(state.node_text(pattern).to_owned(), type_path); return; } @@ -1763,10 +1827,18 @@ impl RustExtractor { /// The nominal type path a type annotation names, seen through references, /// generic arguments, and `dyn`/`impl` trait objects; `None` for tuples, /// slices, function pointers, and anything else without one nominal head. + /// `Self` is the enclosing impl or trait type when one is on the stack. fn stated_type_path(state: &ExtractionState<'_>, ty: TsNode<'_>) -> Option { match ty.kind() { "type_identifier" | "scoped_type_identifier" => { - Some(state.node_text(ty).to_owned()).filter(|path| path != "Self") + let path = state.node_text(ty); + if path == "Self" { + // `Self` in an annotation is the enclosing impl or trait, + // not a type the file declared under that name. + Self::enclosing_receiver_type(state) + } else { + Some(path.to_owned()) + } } "reference_type" | "generic_type" => ty .child_by_field_name("type") diff --git a/crates/tracedecay-code-extraction/tests/main/rust.rs b/crates/tracedecay-code-extraction/tests/main/rust.rs index 9a3a5f7288..664d131ec8 100644 --- a/crates/tracedecay-code-extraction/tests/main/rust.rs +++ b/crates/tracedecay-code-extraction/tests/main/rust.rs @@ -1131,10 +1131,169 @@ fn use_foo() { ref_names.contains(&"Foo::new"), "expected Foo::new call, got: {ref_names:?}" ); - // f.bar() should also produce "bar" (method-name hint). assert!( - ref_names.contains(&"bar"), - "expected 'bar' method-name ref from f.bar(), got: {ref_names:?}" + ref_names.contains(&"f.bar"), + "the receiver-dotted form remains: {ref_names:?}" + ); + // `Foo::new()` does not state that `f` is Foo, and `bar` is the method of + // an impl in this file. Emitting the simple name would invent that caller. + assert!( + !ref_names.contains(&"bar"), + "untyped f.bar() must not emit a bare method name: {ref_names:?}" + ); + assert!( + !ref_names.contains(&"Foo::bar"), + "constructor-like Foo::new() must not fabricate a Foo receiver: {ref_names:?}" + ); +} + +#[test] +fn bare_receiver_calls_name_self_without_the_method_simple_name() { + let source = r#" +fn prepare(value: i32) {} +fn push(value: i32) {} + +struct Rows; +impl Rows { + fn len(&self) -> usize { 0 } + fn measure(&self) -> usize { self.len() } + fn via_explicit(self: &Self) -> usize { self.len() } +} +trait Span {} +impl Span for Rows { + fn wide(&self) -> usize { self.len() } +} + +fn caller(items: Vec, rows: Rows) { + let foreign = make(); + foreign.prepare(1); + items.push(1); + prepare(1); + push(1); + rows.len(); +} +fn make() -> Vec { Vec::new() } +"#; + let result = RustExtractor.extract("src/lib.rs", source); + assert!(result.errors.is_empty(), "{:?}", result.errors); + + let from = |name: &str| { + let function = result + .nodes + .iter() + .find(|node| { + matches!(node.kind, NodeKind::Function | NodeKind::Method) && node.name == name + }) + .unwrap_or_else(|| panic!("{name} is extracted")); + result + .unresolved_refs + .iter() + .filter(|reference| { + reference.reference_kind == EdgeKind::Calls && reference.from_node_id == function.id + }) + .map(|reference| reference.reference_name.as_str()) + .collect::>() + }; + + let measure = from("measure"); + assert!( + measure.contains(&"self.len") && measure.contains(&"Rows::len"), + "{measure:?}" + ); + assert!( + !measure.contains(&"len"), + "self.len() must not emit the bare method name: {measure:?}" + ); + + let via_explicit = from("via_explicit"); + assert!( + via_explicit.contains(&"Rows::len"), + "self: &Self still names the enclosing type: {via_explicit:?}" + ); + + let wide = from("wide"); + assert!( + wide.contains(&"Rows::len"), + "self inside `impl Span for Rows` names Rows, not Span: {wide:?}" + ); + assert!(!wide.contains(&"Span::len"), "{wide:?}"); + + let caller = from("caller"); + assert!(caller.contains(&"prepare"), "{caller:?}"); + assert!(caller.contains(&"push"), "{caller:?}"); + assert!(caller.contains(&"Rows::len"), "{caller:?}"); + assert!(caller.contains(&"Vec::push"), "{caller:?}"); + assert!(caller.contains(&"foreign.prepare"), "{caller:?}"); + assert!(caller.contains(&"items.push"), "{caller:?}"); + assert_eq!( + caller.iter().filter(|name| **name == "prepare").count(), + 1, + "foreign.prepare() invented a second prepare call: {caller:?}" + ); + assert_eq!( + caller.iter().filter(|name| **name == "push").count(), + 1, + "items.push() invented a second push call: {caller:?}" + ); +} + +#[test] +fn self_receiver_names_carry_module_scope_and_the_outer_as_delimiter() { + let source = r#" +mod inner { + pub struct Rows; + impl Rows { + fn len(&self) -> usize { 0 } + fn measure(&self) -> usize { self.len() } + } + trait Wide { fn wide(&self) -> usize; } + impl Wide for Rows { + fn wide(&self) -> usize { self.len() } + } +} +struct Foo; +trait Assoc { type Item; } +trait Local { fn span(&self) -> usize; } +impl Local for ::Item { + fn span(&self) -> usize { self.len() } +} +"#; + let result = RustExtractor.extract("src/lib.rs", source); + assert!(result.errors.is_empty(), "{:?}", result.errors); + + let from = |qualified: &str| { + let function = result + .nodes + .iter() + .find(|node| { + matches!(node.kind, NodeKind::Function | NodeKind::Method) + && node.qualified_name == qualified + }) + .unwrap_or_else(|| panic!("{qualified} is extracted")); + result + .unresolved_refs + .iter() + .filter(|reference| { + reference.reference_kind == EdgeKind::Calls && reference.from_node_id == function.id + }) + .map(|reference| reference.reference_name.as_str()) + .collect::>() + }; + + let measure = from("src/lib.rs::inner::Rows::measure"); + assert!( + measure.contains(&"inner::Rows::len"), + "self inside `mod inner` names the module-scoped type: {measure:?}" + ); + let wide = from("src/lib.rs::inner::::wide"); + assert!( + wide.contains(&"inner::Rows::len"), + "a trait impl in a module keeps the module path: {wide:?}" + ); + let span = from("src/lib.rs::<::Item as Local>::span"); + assert!( + span.contains(&"::Item::len"), + "a projected self type splits at the outer `as`: {span:?}" ); } diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations.rs b/crates/tracedecay-code-index-retention/src/code_index_generations.rs index b4976c6602..103310f3c0 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations.rs @@ -885,6 +885,9 @@ fn plan_code_generation_retention_with_verification_cancellable( Err(error) if error.kind() == std::io::ErrorKind::NotFound && active_pointer.is_none() => { None } + // A pointer is only durable once its generation directory is, so a + // live pointer over an absent directory is loss, not a publisher + // race, and must stay loud. Err(error) => return Err(storage(error)), }; let mut generations = BTreeMap::new(); @@ -1218,7 +1221,7 @@ fn sweep_unreferenced_generation_segments( continue; } let mut reader = CancellableGenerationManifestReaderV1 { - file: File::open(&path).map_err(storage)?, + file: File::open(&path).map_err(deferred_if_absent)?, hasher: Sha256::new(), is_cancelled, cancelled: false, @@ -1281,7 +1284,7 @@ fn sweep_unreferenced_generation_segments( if live_segments.contains(&format!("sha256:{digest}")) { continue; } - let metadata = path.symlink_metadata().map_err(storage)?; + let metadata = path.symlink_metadata().map_err(deferred_if_absent)?; if !metadata.file_type().is_file() { return Err(CodeGenerationRetentionErrorV1::UnsafeState(format!( "generation segment '{}' is not a regular file", @@ -1745,7 +1748,27 @@ fn read_active_pointer( store_root: &Path, ) -> Result { let path = store_root.join(ACTIVE_POINTER_FILE); - let bytes = std::fs::read(&path).map_err(storage)?; + // A directory in the pointer slot makes `read(2)` return EISDIR. That is + // the same corrupt authority the publication store refuses; do not let the + // OS error replace the typed unsafe-state. + match std::fs::metadata(&path) { + Ok(metadata) if metadata.file_type().is_file() => {} + Ok(_) => { + return Err(CodeGenerationRetentionErrorV1::UnsafeState( + "active code-generation pointer is not a regular file".to_owned(), + )); + } + Err(error) => return Err(storage(error)), + } + let bytes = std::fs::read(&path).map_err(|error| { + if error.kind() == std::io::ErrorKind::IsADirectory { + CodeGenerationRetentionErrorV1::UnsafeState( + "active code-generation pointer is not a regular file".to_owned(), + ) + } else { + storage(error) + } + })?; serde_json::from_slice(&bytes).map_err(|error| { CodeGenerationRetentionErrorV1::UnsafeState(format!( "active pointer '{}' is corrupt: {error}", @@ -2033,5 +2056,18 @@ fn storage(error: impl std::fmt::Display) -> CodeGenerationRetentionErrorV1 { CodeGenerationRetentionErrorV1::Storage(error.to_string()) } +/// A path that is not there yet, or that a peer unlinked after this census +/// listed it, is not a broken disk. The publisher creates the scope root and +/// the sealed files under the store lock, then drops that lock; a census that +/// does not hold the lock can observe the gap. The next tick sees a stable +/// tree. Every other I/O failure stays a storage error. +pub(super) fn deferred_if_absent(error: std::io::Error) -> CodeGenerationRetentionErrorV1 { + if error.kind() == std::io::ErrorKind::NotFound { + CodeGenerationRetentionErrorV1::GenerationStoreBusy + } else { + storage(error) + } +} + #[cfg(test)] mod tests; diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/generation_scan.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/generation_scan.rs index 91b997173c..7c2923eaf2 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/generation_scan.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/generation_scan.rs @@ -7,7 +7,8 @@ use tracedecay_domain::canonical_text::{encode_tagged_lowercase_hex, is_lowercas use super::{ CodeGenerationRetentionErrorV1, GenerationDigestVerificationV1, - MAX_GENERATION_METADATA_PREFIX_BYTES, SealedGenerationManifestMetadataV1, storage, + MAX_GENERATION_METADATA_PREFIX_BYTES, SealedGenerationManifestMetadataV1, deferred_if_absent, + storage, }; const MAX_FORMAT_REVISION_PREFIX_BYTES: usize = 4 * 1024; @@ -16,7 +17,7 @@ pub(super) fn read_generation_format_revision( path: &Path, is_cancelled: &dyn Fn() -> bool, ) -> Result { - let mut file = File::open(path).map_err(storage)?; + let mut file = File::open(path).map_err(deferred_if_absent)?; let mut prefix = vec![0_u8; MAX_FORMAT_REVISION_PREFIX_BYTES]; let bytes_read = file.read(&mut prefix).map_err(storage)?; crate::hotpath_observe::retention_inspected(bytes_read as u64); @@ -40,7 +41,7 @@ pub(super) fn read_generation_metadata( is_cancelled: &dyn Fn() -> bool, ) -> Result<(u32, SealedGenerationManifestMetadataV1, String, u64), CodeGenerationRetentionErrorV1> { - let mut file = File::open(path).map_err(storage)?; + let mut file = File::open(path).map_err(deferred_if_absent)?; let size_bytes = file.metadata().map_err(storage)?.len(); let mut hasher = Sha256::new(); let mut prefix = Vec::with_capacity(MAX_GENERATION_METADATA_PREFIX_BYTES); diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/locking.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/locking.rs index 6bdc552abd..1d8d867a75 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/locking.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/locking.rs @@ -102,7 +102,7 @@ fn lock_file( } fn canonical_store_root(store_root: &Path) -> Result { - std::fs::canonicalize(store_root).map_err(storage) + std::fs::canonicalize(store_root).map_err(super::deferred_if_absent) } fn open_lock_file(path: &Path) -> Result { @@ -112,5 +112,5 @@ fn open_lock_file(path: &Path) -> Result { .write(true) .truncate(false) .open(path) - .map_err(storage) + .map_err(super::deferred_if_absent) } diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs index 15ae71fb0e..47d6d406f9 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/tests.rs @@ -1547,6 +1547,23 @@ fn idle_maintenance_preparation_stays_metadata_only() { ); } +#[test] +fn preparation_defers_when_the_scope_root_does_not_exist_yet() { + let parent = tempfile::TempDir::new().expect("parent"); + let missing = parent.path().join("not-created"); + let error = prepare_next_code_generation_retention_cancellable( + &missing, + &BTreeSet::new(), + &|| false, + None, + ) + .expect_err("an unpublished scope root has no census"); + assert!( + matches!(error, CodeGenerationRetentionErrorV1::GenerationStoreBusy), + "a missing scope root is the publisher's create window, not a storage failure: {error:?}" + ); +} + #[test] fn metadata_only_segment_census_observes_at_most_one_directory_entry() { let store = tempfile::TempDir::new().expect("create unpublished store"); @@ -3118,3 +3135,28 @@ fn recovery_completes_a_committed_rewrite_that_never_reached_the_pointer() { plan_code_generation_retention(fixture.store.path(), &BTreeSet::new()) .expect("a recovered store must stay plannable"); } + +/// The census opens every name `read_dir` just returned. Publication can +/// unlink that name first. `NotFound` is the same deferral as a held writer, +/// not a storage failure. Any other open failure stays storage. +#[test] +fn vanished_listed_generation_open_defers_instead_of_storage_loss() { + let root = tempfile::tempdir().expect("census root"); + let missing = root.path().join(format!("generation-{:064x}.json", 1)); + let error = super::generation_scan::read_generation_format_revision(&missing, &|| false) + .expect_err("a vanished listed generation defers the census"); + assert!( + matches!(error, CodeGenerationRetentionErrorV1::GenerationStoreBusy), + "a missing listed generation is a publisher race, not a storage failure: {error:?}" + ); + + let directory = root.path().join("not-a-generation-file"); + std::fs::create_dir(&directory).expect("directory where a file was listed"); + let storage_error = + super::generation_scan::read_generation_format_revision(&directory, &|| false) + .expect_err("a directory is not a vanished file"); + assert!( + matches!(storage_error, CodeGenerationRetentionErrorV1::Storage(_)), + "non-NotFound census I/O stays a storage failure: {storage_error:?}" + ); +} diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/text_artifacts.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/text_artifacts.rs index bc4bfe5588..d56bcbb6ac 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/text_artifacts.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/text_artifacts.rs @@ -198,6 +198,15 @@ fn mutate_verified_text_artifact_under_lock( "publication pointer exceeds its durable byte bound".to_owned(), )); } + // Re-read immediately before the rename. A pointer that is no longer the + // one this mutation observed — including a truncated file — must not be + // replaced by the in-memory copy. + let current = read_active_pointer(store_root)?; + if ¤t != expected_pointer { + return Err(CodeGenerationRetentionErrorV1::Conflict( + "active generation pointer changed before text-artifact mutation".to_owned(), + )); + } atomic_write( &store_root.join(ACTIVE_POINTER_FILE), "code-generation-text-artifact-mutation", diff --git a/crates/tracedecay-code-index-runtime/src/code_index_executor.rs b/crates/tracedecay-code-index-runtime/src/code_index_executor.rs index 0438ee6c2e..a9e79db843 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_executor.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_executor.rs @@ -1501,6 +1501,16 @@ where code_search::CodeIndexSearchUnavailableReasonV1::GenerationUnverified, ); } + // A contended or already-retired staging artifact leaves this + // generation's clone projection unfinished. That is the same + // state `Pending` reports above, so it keeps `Pending`'s + // retryable verdict; `Internal` told callers never to retry a + // window that resolves itself within one background pass. + Err(RetrievalPortError::AuthorityUnavailable(_)) => { + return unavailable( + code_search::CodeIndexSearchUnavailableReasonV1::GenerationUnverified, + ); + } Err(_) => { return unavailable(code_search::CodeIndexSearchUnavailableReasonV1::Internal); } @@ -1697,6 +1707,16 @@ where code_search::CodeIndexSearchUnavailableReasonV1::GenerationUnverified, ); } + // A contended or already-retired staging artifact leaves this + // generation's clone projection unfinished. That is the same + // state `Pending` reports above, so it keeps `Pending`'s + // retryable verdict; `Internal` told callers never to retry a + // window that resolves itself within one background pass. + Err(RetrievalPortError::AuthorityUnavailable(_)) => { + return unavailable( + code_search::CodeIndexSearchUnavailableReasonV1::GenerationUnverified, + ); + } Err(_) => { return unavailable(code_search::CodeIndexSearchUnavailableReasonV1::Internal); } diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs index 3422489f5e..92cf1f063d 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/publication_store.rs @@ -871,6 +871,33 @@ impl DaemonCodeIndexPublicationStoreV1 { CodeIndexPublicationStoreErrorV1::CorruptionResetRequired(error.to_string()) } + /// A pointer slot that is not a regular file is a corrupt authority. + /// + /// `read(2)` and `rename(2)` both report that shape as `EISDIR`. Mapping + /// the OS error to `Unavailable` (or letting it surface as a raw I/O + /// fault) misclassifies a broken publication pointer. Callers in the + /// scheduler publication family must see reset-required corruption. + fn corrupt_non_file_pointer() -> CodeIndexPublicationStoreErrorV1 { + Self::corruption("active code-generation pointer is not a regular file") + } + + fn map_pointer_io(error: std::io::Error) -> CodeIndexPublicationStoreErrorV1 { + if error.kind() == std::io::ErrorKind::IsADirectory { + Self::corrupt_non_file_pointer() + } else { + Self::unavailable(error) + } + } + + fn require_regular_pointer_slot(&self) -> Result<(), CodeIndexPublicationStoreErrorV1> { + match std::fs::metadata(&self.active_path) { + Ok(metadata) if metadata.file_type().is_file() => Ok(()), + Ok(_) => Err(Self::corrupt_non_file_pointer()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(Self::map_pointer_io(error)), + } + } + fn acquire_generation_read_lock( &self, ) -> Result { @@ -1089,8 +1116,11 @@ impl DaemonCodeIndexPublicationStoreV1 { .unwrap_or_else(PoisonError::into_inner) = None; return Ok(None); } - Err(error) => return Err(Self::unavailable(error)), + Err(error) => return Err(Self::map_pointer_io(error)), }; + if !metadata.file_type().is_file() { + return Err(Self::corrupt_non_file_pointer()); + } if metadata.len() > MAX_DURABLE_PUBLICATION_POINTER_BYTES { return Err(Self::corruption( "durable code-generation index exceeds its byte bound", @@ -1102,7 +1132,7 @@ impl DaemonCodeIndexPublicationStoreV1 { // a fixed-width pointer through another path, and a 1-second mtime // filesystem can leave both unchanged while the bytes move. The memo // is reused only when the file digest matches. - let bytes = std::fs::read(&self.active_path).map_err(Self::unavailable)?; + let bytes = std::fs::read(&self.active_path).map_err(Self::map_pointer_io)?; let digest = Self::state_digest(&bytes); { let mut memo = self @@ -1230,38 +1260,98 @@ impl DaemonCodeIndexPublicationStoreV1 { "durable code-generation index exceeds its retention bounds", )); } - *self + let mut memo = self .pointer_memo .lock() - .unwrap_or_else(PoisonError::into_inner) = Some(PublicationPointerMemoV1 { - mtime, - size, - digest, - pointer: pointer.clone(), - }); + .unwrap_or_else(PoisonError::into_inner); + // Install only when the file is still the bytes just parsed. A rename + // that landed during validation owns the memo. + if std::fs::read(&self.active_path).ok().as_deref() == Some(bytes.as_slice()) { + *memo = Some(PublicationPointerMemoV1 { + mtime, + size, + digest, + pointer: pointer.clone(), + }); + } Ok(Some(pointer)) } fn remember_publication_pointer(&self, pointer: &DurablePublicationPointerV1, bytes: &[u8]) { - let metadata = match std::fs::metadata(&self.active_path) { - Ok(metadata) => metadata, - Err(_) => { - *self - .pointer_memo - .lock() - .unwrap_or_else(PoisonError::into_inner) = None; - return; - } - }; - *self + let mut memo = self .pointer_memo .lock() - .unwrap_or_else(PoisonError::into_inner) = Some(PublicationPointerMemoV1 { - mtime: metadata.modified().ok(), - size: metadata.len(), - digest: Self::state_digest(bytes), - pointer: pointer.clone(), - }); + .unwrap_or_else(PoisonError::into_inner); + // The memo and the file it names are one critical section. A publisher + // that observed older bytes must not install them over a newer file. + match std::fs::read(&self.active_path) { + Ok(current) if current == bytes => { + let metadata = std::fs::metadata(&self.active_path).ok(); + *memo = Some(PublicationPointerMemoV1 { + mtime: metadata + .as_ref() + .and_then(|metadata| metadata.modified().ok()), + size: metadata.map_or(0, |metadata| metadata.len()), + digest: Self::state_digest(bytes), + pointer: pointer.clone(), + }); + } + Ok(_) => {} + Err(_) => *memo = None, + } + } + + /// Replace the active pointer only when it is still the exact bytes this + /// publication observed under the store lock. + /// + /// `rename(2)` replaces whatever occupies the path, including a truncated + /// or rewritten pointer. The observation is the compare-and-swap token: + /// a mismatch is a refusal, not a rewrite. `lock` is the witness that + /// this critical section is the exclusive owner of the store. + pub(super) fn commit_observed_pointer( + &self, + _lock: &CodeGenerationStoreLockV1, + observed: Option<&[u8]>, + pointer: &DurablePublicationPointerV1, + bytes: &[u8], + ) -> Result<(), CodeIndexPublicationStoreErrorV1> { + // Refuse a directory (or any non-file) before the read and the + // `rename(2)`. Reading one returns EISDIR, which is not a + // publication-family fault. + self.require_regular_pointer_slot()?; + let current = match std::fs::read(&self.active_path) { + Ok(current) => Some(current), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(Self::unavailable(error)), + }; + if current.as_deref() != observed { + return Err(match current { + Some(current) + if serde_json::from_slice::(¤t).is_err() => + { + Self::corruption("active code-generation pointer is corrupt") + } + _ => CodeIndexPublicationStoreErrorV1::CompareAndSwap, + }); + } + let temporary = self + .active_path + .with_extension(format!("json.{}.tmp", std::process::id())); + if temporary.exists() { + std::fs::remove_file(&temporary).map_err(Self::unavailable)?; + } + Self::write_durable(&temporary, bytes)?; + if let Err(error) = std::fs::rename(&temporary, &self.active_path) { + let _ = std::fs::remove_file(&temporary); + return Err(Self::map_pointer_io(error)); + } + Self::sync_directory( + self.active_path + .parent() + .ok_or_else(|| Self::unavailable("active pointer has no parent directory"))?, + )?; + self.remember_publication_pointer(pointer, bytes); + Ok(()) } pub(super) fn read_retained_partitioned_segment( @@ -2174,6 +2264,15 @@ impl CodeIndexAtomicPublicationPort for DaemonCodeIndexPublicationStoreV1 { } else { self.read_publication_pointer()? }; + // The bytes behind `prior_pointer`, captured under the store lock. + // The commit below refuses to rename unless the file is still these + // exact bytes, so a pointer that changed after this observation is + // not overwritten. + let prior_bytes = if prior_pointer.is_some() { + Some(std::fs::read(&self.active_path).map_err(Self::unavailable)?) + } else { + None + }; if undecoded_expectation.is_none() && prior_pointer .as_ref() @@ -2551,22 +2650,8 @@ impl CodeIndexAtomicPublicationPort for DaemonCodeIndexPublicationStoreV1 { } else { None }; - let temporary = self - .active_path - .with_extension(format!("json.{}.tmp", std::process::id())); - if temporary.exists() { - std::fs::remove_file(&temporary).map_err(Self::unavailable)?; - } hotpath::measure_block!("code_index.generation.publish.pointer_commit", { - Self::write_durable(&temporary, &bytes)?; - std::fs::rename(&temporary, &self.active_path).map_err(Self::unavailable)?; - Self::sync_directory( - self.active_path - .parent() - .ok_or_else(|| Self::unavailable("active pointer has no parent directory"))?, - )?; - self.remember_publication_pointer(&pointer, &bytes); - Ok::<(), CodeIndexPublicationStoreErrorV1>(()) + self.commit_observed_pointer(&_store_lock, prior_bytes.as_deref(), &pointer, &bytes) })?; drop(source_fence); let mut state = self.cache.lock_state()?; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index c2c7eecdde..b686c00dc6 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -608,6 +608,37 @@ impl SourceFreshnessFenceV1 { }) && self.snapshot_is_recently_verified(&state, project_root, shutting_down) } + + /// Whether the last completed proof was sealed from exactly this snapshot. + /// + /// Clock age is not part of the answer. A seal or clone backfill can + /// outlive the admission window without the snapshot changing identity. + pub(super) fn proof_describes_snapshot( + &self, + snapshot_content_identity: &ContentDigest, + ) -> bool { + let state = self.snapshot(); + state.verified_against_source + && state.source_witness.as_ref().is_some_and(|witness| { + witness + .content_manifest + .describes_snapshot(snapshot_content_identity) + }) + } + + /// Refresh the admission clock and the git-metadata sample after the + /// sealed digests still matched. The content witness and reconciled + /// epoch stay put: this is the same proof, not a new generation. + fn rebind_admission_clock(&self, git_metadata: identity::GitMetadataFingerprintV1) { + let micros = now_micros().0; + let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + state.git_metadata = git_metadata; + state.last_reconciled_at = Instant::now(); + state.verified_against_source = true; + state.freshness_unknown = false; + self.last_reconciled_at_micros + .store(micros, Ordering::Release); + } } /// What the cheap Git/stat freshness ladder concluded about the retained @@ -1737,6 +1768,45 @@ impl CodeIndexWorktreeSchedulerV1 { Ok(Some(outcome)) } + /// Record that `metadata` is the generation the live worktree still seals. + /// + /// The in-memory fence takes this snapshot. The disk witness, when one + /// exists, is rewritten to this generation id so the next open does not + /// treat the predecessor's proof as a reason to drop it and reseal. + fn accept_unchanged_sealed_snapshot( + &mut self, + metadata: &VerifiedSealedTextGenerationMetadataV1, + git_metadata: identity::GitMetadataFingerprintV1, + stat_signature: String, + source_manifest: SourceContentManifestV1, + prior_witness: Option<&RestoreFreshnessWitnessV1>, + ) -> CodeIndexReconcileOutcomeV1 { + let snapshot_content_identity = metadata.snapshot().content_identity.clone(); + self.latest_content_identity = Some(snapshot_content_identity.clone()); + self.mark_reconciled_retained_generation_state( + git_metadata.clone(), + Some(ReconciledSourceWitnessV1 { + stat_signature: stat_signature.clone(), + content_manifest: source_manifest, + }), + ); + if let Some(prior) = prior_witness { + RestoreFreshnessWitnessV1 { + generation_id: metadata.manifest().generation_id.as_str().to_owned(), + git_metadata_signature: git_metadata.stable_signature(), + stat_signature, + repository_parse_identity_digest: prior.repository_parse_identity_digest.clone(), + ignored_source_admissions_digest: prior.ignored_source_admissions_digest.clone(), + ignored_source_paths: Vec::new(), + } + .persist(&self.store_root); + } + CodeIndexReconcileOutcomeV1::Noop(CodeIndexNoopEvidenceV1 { + snapshot_content_identity, + overflow_reconciled: false, + }) + } + pub(super) fn reconcile_retained_text_generation_with( &mut self, metadata: &VerifiedSealedTextGenerationMetadataV1, @@ -1758,10 +1828,14 @@ impl CodeIndexWorktreeSchedulerV1 { .observe_retained_text_compatibility(metadata) .is_reusable(); let witness = RestoreFreshnessWitnessV1::load(&self.store_root); - if witness.as_ref().is_some_and(|witness| { - witness.generation_id != metadata.manifest().generation_id.as_str() - || !witness.ignored_source_paths.is_empty() - }) || !self.ignored_source_admissions.is_empty() + // A predecessor freshness witness is not a reason to drop this + // generation. It names the proof that sealed an earlier snapshot. + // Ignored-source rosters still require the complete capture: their + // digest is not the ordinary file manifest this path compares. + if witness + .as_ref() + .is_some_and(|witness| !witness.ignored_source_paths.is_empty()) + || !self.ignored_source_admissions.is_empty() { return Ok(None); } @@ -1788,37 +1862,54 @@ impl CodeIndexWorktreeSchedulerV1 { // generation's sealed file digests; its matching stat signature is // the negative cache that lets a moved tree skip the byte comparison. let source_manifest = SourceContentManifestV1::for_snapshot(metadata.snapshot()); - if retained_is_reusable + let sealed_bytes_match = retained_is_reusable && !has_hints - && let Some(witness) = witness.as_ref() - && witness.git_metadata_signature == sampled_metadata.stable_signature() - && witness.stat_signature == sampled_sweep.signature && sampled_sweep.content_matches( &self.project_root, &source_manifest, &self.shutting_down, - ) + ); + let quiet_witness = sealed_bytes_match + && witness.as_ref().is_some_and(|witness| { + witness.git_metadata_signature == sampled_metadata.stable_signature() + && witness.stat_signature == sampled_sweep.signature + }); + // Identical source bytes do not make a moved commit or branch the same + // generation. `finish_retained_reconcile` rebuilds on exactly this + // drift, and branch-scoped reads resolve generations by their sealed + // `source_revision`, so accepting here would leave the retained + // generation attributed to a commit the checkout has left for as long + // as the bytes hold still. `self.identity` was re-resolved above, so + // this costs no extra walk. A snapshot sealed without a revision + // (a dirty capture) has no commit attribution to invalidate. + let sealed_attribution_is_current = metadata.snapshot().reference.as_ref() + == self.identity.head_ref() + && metadata + .snapshot() + .source_revision + .as_ref() + .is_none_or(|sealed| self.identity.head_commit() == Some(sealed)); + // Graph-on refuses to decode the sealed generation just because the + // predecessor witness, or a git-index mtime this seal itself moved, + // does not name this generation. The sealed digests are the proof. + // Graph-off still captures so a metadata-only drift is verified + // without a full decode when the quiet witness is absent. + if sealed_bytes_match + && sealed_attribution_is_current + && (quiet_witness || !rebuild_changed_source_without_decode) { - let snapshot_content_identity = metadata.snapshot().content_identity.clone(); - self.latest_content_identity = Some(snapshot_content_identity.clone()); - self.mark_reconciled_retained_generation_state( + return Ok(Some(self.accept_unchanged_sealed_snapshot( + metadata, sampled_metadata, - Some(ReconciledSourceWitnessV1 { - stat_signature: sampled_sweep.signature, - content_manifest: source_manifest, - }), - ); - return Ok(Some(CodeIndexReconcileOutcomeV1::Noop( - CodeIndexNoopEvidenceV1 { - snapshot_content_identity, - overflow_reconciled: false, - }, + sampled_sweep.signature, + source_manifest, + witness.as_ref(), ))); } - // A compatible generation whose witness did not prove a quiet tree - // falls through to the full graph-on reconcile. An incompatible - // lightweight owner rebuilds here without decoding the retained graph. + // A compatible generation whose bytes moved falls through to the full + // graph-on reconcile. An incompatible lightweight owner rebuilds here + // without decoding the retained graph. if retained_is_reusable && !rebuild_changed_source_without_decode { return Ok(None); } @@ -2813,6 +2904,50 @@ impl CodeIndexWorktreeSchedulerV1 { .source_currency_witness_for(generation_id, snapshot_content_identity) } + /// Bind a sealed snapshot to the source proof, renewing an expired clock + /// when the sealed digests still match. + /// + /// The admission window is 30s. This does not move the clone-successor + /// copy off the publication advance. It only stops an expired clock, or a + /// predecessor disk witness, from clearing the generation those digests + /// already name. A hook epoch or a digest mismatch still refuses. + pub(super) fn currency_witness_for_sealed_snapshot( + &self, + generation_id: &CodeGenerationId, + snapshot_content_identity: &ContentDigest, + ) -> Option { + if self.shutting_down.load(Ordering::Acquire) { + return None; + } + if self.freshness_fence.serves_recently_verified_source( + snapshot_content_identity, + &self.project_root, + &self.shutting_down, + ) { + return self + .freshness_fence + .source_currency_witness_for(generation_id, snapshot_content_identity); + } + if !self + .freshness_fence + .proof_describes_snapshot(snapshot_content_identity) + || self.freshness_fence.source_change_pending() + { + return None; + } + let freshness = self.freshness_fence.snapshot(); + if !self.source_witness_matches_worktree(&freshness) { + return None; + } + // Sample after the walk. `gix::open` inside the digest comparison can + // move index metadata; storing the post-walk sample is what keeps the + // next probe from calling that side effect a new generation. + let git_metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); + self.freshness_fence.rebind_admission_clock(git_metadata); + self.freshness_fence + .source_currency_witness_for(generation_id, snapshot_content_identity) + } + /// A cheap stat-level (path, mtime, size) signature of the present source /// candidates. It opens gix and runs stat-based status (no byte reads, no /// content hashing). A changed signature skips straight to reconcile; an @@ -3252,6 +3387,19 @@ impl CodeIndexWorktreeSchedulerV1 { self.publication.sealed_decode_count() } + /// Age the admission clock past its own threshold without touching source. + #[cfg(test)] + pub(super) fn expire_source_proof_for_test(&self) { + let mut state = self + .freshness_fence + .state + .lock() + .unwrap_or_else(PoisonError::into_inner); + state.last_reconciled_at = Instant::now() + .checked_sub(state.staleness_threshold + Duration::from_secs(1)) + .unwrap_or_else(Instant::now); + } + #[cfg(any(test, feature = "test-helpers"))] pub fn poison_decoded_publication_cache_for_test(&self) { self.publication.poison_decoded_cache_for_test(); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry.rs index a5392c83b1..ab44191731 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry.rs @@ -2109,6 +2109,21 @@ impl CodeIndexSchedulerRegistryV1 { } } + /// Stamp a continuation while `reconcile_in_progress` still reports this pass. + /// + /// Callers that already released the worker's pass guard use this so a + /// reader waiting for the counter to hit zero cannot observe an empty + /// slot and then lose to `BusyFollowUp`. The stamp is the idle boundary; + /// the guard lives only for the note. + fn note_visible_worker_continuation( + passes: &Arc, + pending_wake: &PendingWakeV1, + wake: &tokio::sync::Notify, + ) { + let _visible = super::ReconcilePassGuard::enter(passes); + Self::note_worker_continuation(pending_wake, wake); + } + /// Claim the pending wake as one reconcile's arrival, at the instant the /// scheduler dequeues it. /// diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs index ac239ce49c..ce5dc01754 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs @@ -1033,10 +1033,16 @@ impl CodeIndexSchedulerRegistryV1 { } // Source reconciliation is complete: release the background // admission permit before HeadOpening / graph work so sibling - // stores can start. Keep `reconcile_pass` through text - // seating, dropping it made `reconcile_in_progress` lie while - // this worker still owned graph try_lock, which deadlocked - // tests that hold the scheduler mutex and wait for that flag. + // stores can start. The permit is never re-acquired inside + // this pass: `_build_publication` is held for the rest of the + // iteration, and `run_ignored_dependency_admission` takes the + // admission *before* that same gate, so waiting on admission + // here would invert that order (see + // `background_worker_waits_for_global_admission_before_publication_gate`). + // Keep `reconcile_pass` through text seating, dropping it + // made `reconcile_in_progress` lie while this worker still + // owned graph try_lock, which deadlocked tests that hold the + // scheduler mutex and wait for that flag. drop(_background_reconcile_admission); // A publication must first reopen its own lightweight text // owner: publication moved the durable pointer, so the prior @@ -1110,6 +1116,12 @@ impl CodeIndexSchedulerRegistryV1 { && !graph_activation_deferred && let Some(text) = graph_text.clone() { + // `reconcile_pass` is held across this projection, so + // the pointer rename is inside the pass a reader + // samples. Taking the admission permit back here + // instead would deadlock against an + // ignored-dependency owner that already holds it and + // is waiting for `_build_publication`. let projection = tokio::spawn(Self::drive_text_projection( text, Arc::clone(&worker_shutting_down), @@ -1168,7 +1180,21 @@ impl CodeIndexSchedulerRegistryV1 { // A successor-only retained projection holds no pass guard of // its own; keeping the worker's guard through graph seat would // report rebuild_in_flight for clone backfill that is not - // exact/lexical work. + // exact/lexical work. Stamp the continuation this projection + // already owes before that drop: the slot, not a later note, + // is what an idle reader observes. + if let Some(outcome) = published_text_projection_outcome.as_ref() { + let schedule_continuation = match outcome { + PublishedTextProjectionOutcomeV1::Finished => graph_text + .as_ref() + .is_some_and(LatestCodeTextGenerationV1::text_projection_needs_work), + PublishedTextProjectionOutcomeV1::Unfinished => true, + PublishedTextProjectionOutcomeV1::Shutdown => false, + }; + if schedule_continuation { + Self::note_worker_continuation(&worker_pending_wake, &worker_wake); + } + } if retained_text_projection.is_none() || retained_projection_successor_only { drop(reconcile_pass.take()); } @@ -1296,6 +1322,14 @@ impl CodeIndexSchedulerRegistryV1 { .filter(|retained| retained.uses_partitioned_manifest()) .cloned() { + // Every outcome of this attempt schedules one successor. + // Stamp it before the recovery await, while the pass is + // visible, so the wait cannot be sampled as an idle slot. + Self::note_visible_worker_continuation( + &worker_reconcile_in_progress, + &worker_pending_wake, + &worker_wake, + ); retained_graph_head_recovery_attempted = true; let generation_id = retained.metadata().manifest().generation_id.clone(); let replay_scheduler = Arc::clone(&worker_scheduler); @@ -1400,8 +1434,8 @@ impl CodeIndexSchedulerRegistryV1 { // all and never published the successor generation. The // `retained_graph_head_recovery_attempted` guard above is // now false for every later pass, so this cannot spin - // another retained-recovery Noop. - Self::note_worker_continuation(&worker_pending_wake, &worker_wake); + // another retained-recovery Noop. The successor was + // stamped before this await. } // A recovered revision-7 verified head already serves its // native graph from the retained text owner, and that owner @@ -1458,6 +1492,8 @@ impl CodeIndexSchedulerRegistryV1 { let graph_text = graph_text.clone(); let shutting_down = Arc::clone(&worker_shutting_down); let prepare_passes = Arc::clone(&worker_reconcile_in_progress); + let prepare_pending_wake = Arc::clone(&worker_pending_wake); + let prepare_wake = Arc::clone(&worker_wake); match hotpath::future!( tokio::task::spawn_blocking(move || { let decoder = Self::lock_scheduler_for_graph_step( @@ -1513,6 +1549,18 @@ impl CodeIndexSchedulerRegistryV1 { )? .1 .take_ignored_roster_refusal_rebuild(); + if roster_refusal_rebuild { + // One pass, claimed from the scheduler, so + // a refusal that keeps reproducing cannot + // spin this worker. Stamp before this + // closure drops the step guard: the result + // is observed only after the slot is set. + Self::note_visible_worker_continuation( + &prepare_passes, + &prepare_pending_wake, + &prepare_wake, + ); + } let replay_binding = match latest.as_ref() { Some(latest) => Some( Self::lock_scheduler_for_graph_step( @@ -1545,15 +1593,6 @@ impl CodeIndexSchedulerRegistryV1 { the sealed generation cannot seat" ); } - if roster_refusal_rebuild { - // One pass, claimed from the scheduler, so - // a refusal that keeps reproducing cannot - // spin this worker. - Self::note_worker_continuation( - &worker_pending_wake, - &worker_wake, - ); - } Ok((outcome, latest, replay_binding)) } Ok(Err(error)) => { @@ -1714,7 +1753,14 @@ impl CodeIndexSchedulerRegistryV1 { .as_ref() .is_some_and(LatestCodeTextGenerationV1::text_projection_needs_work) { - Self::note_worker_continuation(&worker_pending_wake, &worker_wake); + // Already stamped before optional graph. Re-enter + // the pass so a reader that cleared the slot + // during graph still cannot sample the stamp. + Self::note_visible_worker_continuation( + &worker_reconcile_in_progress, + &worker_pending_wake, + &worker_wake, + ); } // Large text projections can outlive the bounded // source proof established before publication. The @@ -1784,7 +1830,11 @@ impl CodeIndexSchedulerRegistryV1 { "the publication's text owner did not finish its projection; \ the sealed generation stays unseated until it does" ); - Self::note_worker_continuation(&worker_pending_wake, &worker_wake); + Self::note_visible_worker_continuation( + &worker_reconcile_in_progress, + &worker_pending_wake, + &worker_wake, + ); } } // Keep the pass lifetime around the post-projection source @@ -1801,8 +1851,6 @@ impl CodeIndexSchedulerRegistryV1 { let text_generation = Arc::clone(&worker_text_generation); let serving_seats = Arc::clone(&worker_serving_seats); let serving_generation_changed = worker_serving_generation_changed.clone(); - let source_freshness = worker_source_freshness.clone(); - let project_root = worker_project_root.clone(); let text_latest = latest.clone(); let latest = latest.clone(); let shutting_down = Arc::clone(&worker_shutting_down); @@ -1836,13 +1884,15 @@ impl CodeIndexSchedulerRegistryV1 { // proofs to the seat. Asking the fence whether it // has verified *this* sealed snapshot is what makes // the binding truthful for a seat this pass did not - // publish. - let pass_proves_latest = source_freshness - .serves_recently_verified_source( - &latest.generation().snapshot().content_identity, - &project_root, - &shutting_down, - ); + // publish. An expired clock, or a git-index sample + // this seal moved, is not a different snapshot. + // Dropping the witness here cleared the newer + // generation. The lexical full-copy is not decided + // on this swap. + let sealed_currency = scheduler.currency_witness_for_sealed_snapshot( + &latest.generation().manifest().generation_id, + &latest.generation().snapshot().content_identity, + ); let mut serving = serving_generation .write() .unwrap_or_else(std::sync::PoisonError::into_inner); @@ -1883,17 +1933,7 @@ impl CodeIndexSchedulerRegistryV1 { *serving_source_witness .write() .unwrap_or_else(std::sync::PoisonError::into_inner) = - pass_proves_latest - .then(|| { - source_freshness.source_currency_witness_for( - &latest.generation().manifest().generation_id, - &latest - .generation() - .snapshot() - .content_identity, - ) - }) - .flatten(); + sealed_currency; } // The durable pointer names a successor, so no // proof of this seat's currency exists to bind. @@ -1965,7 +2005,11 @@ impl CodeIndexSchedulerRegistryV1 { if text_latest.text_projection_needs_work() && !text_latest.query_owners_are_ready() { - Self::note_worker_continuation(&worker_pending_wake, &worker_wake); + Self::note_visible_worker_continuation( + &worker_reconcile_in_progress, + &worker_pending_wake, + &worker_wake, + ); } } Ok(Err(error)) => { @@ -1994,7 +2038,27 @@ impl CodeIndexSchedulerRegistryV1 { } // The source proof and serving witness are now published as // one lifecycle. Optional receipts do not keep source - // verification in flight. + // verification in flight. A clone-backfill continuation this + // pass already knows about is stamped first, so the drop is + // not an empty slot. + if clone_backfill_waiting_for_source + && matches!( + &result, + Ok((Ok(CodeIndexReconcileOutcomeV1::Noop(_)), _, _)) + ) + && worker_text_generation + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .is_some() + && worker_source_freshness + .ready_without_stat(&worker_project_root, &worker_shutting_down) + { + Self::note_visible_worker_continuation( + &worker_reconcile_in_progress, + &worker_pending_wake, + &worker_wake, + ); + } drop(reconcile_pass.take()); if let Ok((Ok(outcome), _, _)) = &result { // A pass that ran to a terminal outcome proves neither the @@ -2046,12 +2110,8 @@ impl CodeIndexSchedulerRegistryV1 { ); } worker_serving_generation_changed.send_replace(()); - // The retained slice was checked before reconciliation - // renewed this proof. Preserve its wake now that source - // is current, without requiring another query arrival. - if clone_backfill_waiting_for_source { - Self::note_worker_continuation(&worker_pending_wake, &worker_wake); - } + // The clone-backfill continuation was stamped before + // this pass dropped `reconcile_in_progress`. } } else { // Surface bounded non-terminal failure without new project-path data. @@ -2269,7 +2329,6 @@ impl CodeIndexSchedulerRegistryV1 { PublishedTextProjectionOutcomeV1::Unfinished } }; - drop(reconcile_pass.take()); match outcome { PublishedTextProjectionOutcomeV1::Finished if !retained_head_recovered_without_complete_replay @@ -2316,6 +2375,9 @@ impl CodeIndexSchedulerRegistryV1 { Self::note_worker_continuation(&worker_pending_wake, &worker_wake); } } + // The continuation is already in the slot. Dropping here + // is the first moment this pass looks idle. + drop(reconcile_pass.take()); } if worker_shutting_down.load(Ordering::Acquire) { tracing::info!( diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/serving_reads.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/serving_reads.rs index 668576b8d5..2c9a5b0a36 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/serving_reads.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/serving_reads.rs @@ -564,18 +564,16 @@ impl CodeIndexSchedulerRegistryV1 { .read() .unwrap_or_else(std::sync::PoisonError::into_inner) .clone(); - // A still-current proof needs no follow-up. If it expired - // after this pass began, leave one coalesced wake so the - // worker re-observes source after releasing its ownership. - if serving.is_some() - && !source_freshness.ready_without_stat(&freshness_root, &shutting_down) - { - Self::note_wake_if_idle( - &pending_wake, - &wake, - CodeIndexCadenceTriggerV1::BusyFollowUp, - ); - } + // The holder of the scheduler is already the source + // observation. A follow-up posted from this read is taken + // by that pass, the slot goes empty, and the next poll + // finds the lock still held with the proof not yet + // renewed and posts another. Dashboard freshness reads + // that slot as `refresh_in_flight` and stays `Verifying` + // for the whole chain. The pass renews the proof before + // it releases the lock; a proof that is still expired + // afterwards is requested by the next read that acquires + // the scheduler. return serving; } }; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs index 394cc15419..cb082cfd60 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs @@ -1312,15 +1312,29 @@ impl DaemonCodeTextArtifactStoreV1 { } let _lock = self.acquire_store_write_lock()?; checkpoint_text_artifact_control(control)?; - let metadata = staging_path - .symlink_metadata() - .map_err(text_artifact_unavailable)?; - if !metadata.file_type().is_file() { - return Err(RetrievalPortError::Contract( - "incompatible text-artifact staging path is not a regular file".to_owned(), - )); + match staging_path.symlink_metadata() { + Ok(metadata) if metadata.file_type().is_file() => { + retire_text_artifact_staging_family(staging_path) + .map_err(text_artifact_unavailable)?; + } + Ok(_) => { + return Err(RetrievalPortError::Contract( + "incompatible text-artifact staging path is not a regular file".to_owned(), + )); + } + // A concurrent build may retire this staging file first. Discard + // wants it gone, so finding it already gone is the end state, not + // an unavailable authority: reporting one aborts the caller's + // reopen and the clone lane answers a non-retryable failure for a + // state that has already resolved. Sidecars can outlive the + // database after a crash, so sweep them the way + // `prepare_absent_text_artifact_staging` does. + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + clear_text_artifact_staging_sidecars(staging_path) + .map_err(text_artifact_unavailable)?; + } + Err(error) => return Err(text_artifact_unavailable(error)), } - retire_text_artifact_staging_family(staging_path).map_err(text_artifact_unavailable)?; DaemonCodeIndexPublicationStoreV1::sync_directory(&artifacts_root) .map_err(text_artifact_unavailable) } @@ -3273,7 +3287,7 @@ impl LatestCodeTextGenerationV1 { )); }; drop(slot); - let mut publish_claim = TextHeadOpenClaimV1::new(&self.text_projection_build); + let _publish_claim = TextHeadOpenClaimV1::new(&self.text_projection_build); let CodeTextArtifactBuildV1 { builder, source, @@ -3312,7 +3326,6 @@ impl LatestCodeTextGenerationV1 { ) .map_err(map_text_artifact_error)?; let needs_clone_successor = !reader.has_clone_fingerprints(); - let prior = reader.verified_artifact().clone(); // Match the cold-open path: install owners first, then publish Ready. // Publishing Ready before a failed install (admission ceiling / shrink) // would leave dashboard/MCP progress claiming a ready generation that @@ -3320,10 +3333,16 @@ impl LatestCodeTextGenerationV1 { self.install_artifact_owners(reader, reader_reservation)?; self.publish_text_progress_phase(CodeIndexBuildPhaseV1::Ready, 0, 0); if needs_clone_successor { - let source = store.open_sealed_source(&sealed_identity, control)?; - let build = - self.begin_clone_successor(descriptor, prior, sealed_identity, source, control)?; - drop(publish_claim.install(TextHeadOpenBuildV1::CloneSuccessor(build))); + // `begin_clone_successor` copies the whole prior lexical artifact + // before the first page walk. Doing that here kept this advance, + // and the publication pass awaiting it, inside `reconcile_in_progress` + // for the copy. Exact and lexical serving are already installed; + // the copy is not a freshness precondition. Leave the slot pending + // so the retained driver starts the successor after the seat, + // without the receipt guard. The claim stays armed: its drop + // restores only `HeadOpening`, so `CloneSuccessorPending` survives + // and parked wakes are notified. + self.text_projection_build.retain_clone_successor_retry()?; return Ok(false); } Ok(true) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs index c82155ff9b..8cf50ea13c 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/mod.rs @@ -1338,14 +1338,11 @@ async fn settled_owner_with_idle_admission( /// done disturbing it. /// /// The caller must already hold the single background admission, so no further -/// pass can start. One pass can still be finishing: the worker releases that -/// admission halfway through its body and drops its `reconcile_pass` guard -/// before the branches that call `note_worker_continuation`, so both -/// `reconcile_in_progress` and the slot read quiet while the tail is still -/// about to stamp `BusyFollowUp` into it. [`wait_for_settled_owner`] samples -/// exactly those two, so it cannot see that tail. With the admission held the -/// tail is finite and unrepeatable, so clearing until the slot survives a quiet -/// window is the proof the settle cannot give. +/// pass can start. A pass stamps `BusyFollowUp` before it drops +/// `reconcile_in_progress`, but a notify already banked by that pass can still +/// be claimed the moment the permit is released. Clearing until the slot +/// survives a quiet window is the proof the settle cannot give once that +/// release is the next thing that happens. async fn clear_pending_wake_until_quiet( registry: &CodeIndexSchedulerRegistryV1, scope: &tracedecay_contracts::ResolvedScope, diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/publication_store.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/publication_store.rs index 95f3f7a4d7..f5285d04d0 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/publication_store.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/publication_store.rs @@ -2648,3 +2648,58 @@ fn publication_pointer_memo_follows_bytes_when_size_and_mtime_stay_put() { "equal size and mtime must not reuse the previous pointer" ); } + +#[test] +fn stale_pointer_commit_does_not_replace_a_changed_active_pointer() { + let store = TempDir::new().expect("store root"); + let project = TempDir::new().expect("project root"); + let publication = super::super::DaemonCodeIndexPublicationStoreV1::new( + store.path(), + project.path(), + SanitizerRevision::new(tracedecay_privacy::CODE_SOURCE_SANITIZER_VERSION_V1) + .expect("sanitizer revision"), + ) + .expect("open publication store"); + let pointer_path = store.path().join("active-code-generation-v1.json"); + let observed = same_length_publication_pointer("generation.observed", 0x31); + let observed_bytes = serde_json::to_vec(&observed).expect("encode observed pointer"); + let replacement = same_length_publication_pointer("generation.replacement", 0x32); + let replacement_bytes = serde_json::to_vec(&replacement).expect("encode replacement pointer"); + std::fs::write(&pointer_path, &observed_bytes).expect("write observed pointer"); + let store_lock = acquire_code_generation_store_lock(store.path()).expect("store lock"); + + publication + .commit_observed_pointer( + &store_lock, + Some(&observed_bytes), + &replacement, + &replacement_bytes, + ) + .expect("matching observation publishes"); + assert_eq!( + std::fs::read(&pointer_path).expect("published pointer"), + replacement_bytes + ); + + std::fs::write(&pointer_path, b"{").expect("truncate active pointer"); + let error = publication + .commit_observed_pointer( + &store_lock, + Some(&replacement_bytes), + &observed, + &observed_bytes, + ) + .expect_err("a stale observation must not publish"); + assert!( + matches!( + error, + CodeIndexPublicationStoreErrorV1::CorruptionResetRequired(_) + ), + "corrupt pointer is a closed publication failure, not a rewrite: {error:?}" + ); + assert_eq!( + std::fs::read(&pointer_path).expect("faulted pointer remains"), + b"{", + "the truncated pointer must still be the file" + ); +} diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs index 188d869742..72b2c508cf 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs @@ -851,7 +851,7 @@ fn retained_stale_rust_extractor_generation_is_refused_and_rebuilt() { .iter() .find(|(language, _)| language.as_str() == "rust") .map(|(_, revision)| revision.as_str()), - Some("extractor.rust.v10") + Some("extractor.rust.v11") ); } @@ -3772,6 +3772,144 @@ async fn dashboard_progress_does_not_wait_for_the_scheduler_mutex() { registry.shutdown().await; } +/// A query that cannot join the owner must not schedule the verification the +/// dashboard would then report as `Verifying`. The in-flight pass renews an +/// expired proof before it releases the scheduler; a read that posts +/// `BusyFollowUp` while that pass holds the lock is taken and immediately +/// replaced by the next poll, so the ladder never settles to `Fresh`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn busy_query_does_not_rearm_dashboard_verification() { + let fixture = GitFixture::new(&[("src/main.rs", "fn main() {}\n")]); + let store = TempDir::new().expect("store root"); + let registry = CodeIndexSchedulerRegistryV1::with_background_reconcile_permits(1, 1); + registry + .mount_worktree( + test_project_id(), + fixture.path(), + store.path().to_path_buf(), + ) + .await + .expect("mount daemon-owned scheduler"); + wait_for_initial_generation(®istry, fixture.path()).await; + wait_for_dashboard_ready(®istry, fixture.path()).await; + drain_clone_backfill(®istry, fixture.path()).await; + settled_owner_with_idle_admission(®istry, fixture.path()).await; + let admission = quiesced_background_reconcile_admission(®istry, fixture.path()).await; + let canonical_root = fixture + .path() + .canonicalize() + .expect("canonical fixture root"); + let scope = { + let mounted = registry.mounted.lock().await; + let worktree = mounted.get(&canonical_root).expect("mounted worktree"); + tracedecay_contracts::ResolvedScope::new( + test_project_id(), + worktree.repository_id.clone(), + worktree.worktree_id.clone(), + None, + ) + .expect("resolved scope") + }; + clear_pending_wake_until_quiet(®istry, &scope).await; + let freshness = registry + .source_freshness_for_root(fixture.path()) + .await + .expect("mounted freshness fence"); + { + let mut state = freshness.state.lock().expect("freshness state"); + state.last_reconciled_at = Instant::now() + .checked_sub(state.staleness_threshold + Duration::from_secs(1)) + .expect("age the readiness proof"); + } + let scheduler = { + let mounted = registry.mounted.lock().await; + Arc::clone( + &mounted + .get(&canonical_root) + .expect("mounted worktree") + .scheduler, + ) + }; + let (locked_tx, locked_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let scheduler_holder = tokio::task::spawn_blocking(move || { + let _scheduler_guard = scheduler.lock().expect("hold scheduler mutex"); + let _ = locked_tx.send(()); + let _ = release_rx.blocking_recv(); + }); + locked_rx.await.expect("scheduler mutex holder started"); + for _ in 0..8 { + assert!( + registry + .latest_complete_fresh(fixture.path()) + .await + .is_some(), + "a busy owner still serves the seated generation" + ); + } + assert_eq!( + registry.pending_wake_micros_for_root(fixture.path()).await, + Some(0), + "a read blocked on the in-flight owner must not schedule another verification" + ); + let projected = registry + .dashboard_freshness(fixture.path()) + .await + .expect("dashboard freshness while the owner holds the scheduler"); + assert_eq!( + projected.staleness_state, + Some(tracedecay_contracts::code_index_freshness::CodeIndexStalenessStateV1::Fresh), + "an owner that has not observed a source change is not Verifying" + ); + let _ = release_tx.send(()); + scheduler_holder + .await + .expect("scheduler mutex holder joined"); + + assert!( + registry + .latest_complete_fresh(fixture.path()) + .await + .is_some(), + "the seated generation remains servable once the owner releases the scheduler" + ); + assert!( + registry + .pending_wake_micros_for_root(fixture.path()) + .await + .is_some_and(|pending| pending != 0), + "an uncontended read of an expired proof still requests one verification" + ); + drop(admission); + tokio::time::timeout(SERVING_SEAT_FAILURE_CEILING, async { + loop { + let settled = registry + .dashboard_freshness(fixture.path()) + .await + .is_some_and(|freshness| { + freshness.staleness_state + == Some( + tracedecay_contracts::code_index_freshness::CodeIndexStalenessStateV1::Fresh, + ) + }) + && registry + .pending_wake_micros_for_root(fixture.path()) + .await + == Some(0) + && !registry + .reconcile_in_progress_for_test(fixture.path()) + .await; + if settled { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("the single verification settles back to Fresh"); + registry.shutdown().await; +} + // Two workers so the timeout timer stays live if a regression parks one // runtime worker on the scheduler mutex: the test then fails instead of // deadlocking against its own release channel. @@ -8173,6 +8311,139 @@ fn graph_off_stale_witness_reconciles_unchanged_source_without_full_decode() { ); } +/// The disk freshness witness names whichever generation last persisted it. +/// A later seal of the same bytes used to return `None` the moment that id +/// disagreed, and the graph-on caller then decoded and resealed. Under load +/// that reseal outlived the admission window, the swap cleared the witness, +/// and the newer generation never became current. Unchanged sealed bytes +/// keep the generation and rewrite the witness onto it. Moved bytes still +/// refuse, without publishing a substitute. +#[test] +fn predecessor_freshness_witness_keeps_the_sealed_generation() { + let fixture = GitFixture::new(ALPHA_LIB_V1); + let store = TempDir::new().expect("store root"); + let mut scheduler = scheduler( + &fixture, + store.path().to_path_buf(), + Arc::new(SharedCodeIndexBytePoolV1::default()), + ); + let seeded = published(scheduler.reconcile_now().expect("seed retained generation")); + let metadata = scheduler + .servable_retained_text_generation() + .expect("publication store") + .expect("authenticated retained text generation") + .metadata() + .clone(); + let generation_id = metadata.manifest().generation_id.clone(); + let mut witness = + RestoreFreshnessWitnessV1::load(store.path()).expect("the seal persisted a proof"); + assert_eq!(witness.generation_id, generation_id.as_str()); + witness.generation_id = "generation.predecessor".to_owned(); + witness.persist(store.path()); + let index_path = fixture.path().join(".git/index"); + let index_mtime = std::fs::metadata(&index_path) + .expect("git index metadata") + .modified() + .expect("git index mtime"); + filetime::set_file_mtime( + &index_path, + filetime::FileTime::from_system_time(index_mtime + Duration::from_secs(2)), + ) + .expect("advance only the git index mtime"); + + let decodes_before = scheduler.sealed_decode_count(); + let outcome = scheduler + .reconcile_retained_text_generation_with(&metadata, false) + .expect("graph-on retained reconcile") + .expect("unchanged sealed bytes must not be dropped"); + let CodeIndexReconcileOutcomeV1::Noop(evidence) = outcome else { + panic!("predecessor proof must not reseal the same snapshot: {outcome:?}"); + }; + assert_eq!( + evidence.snapshot_content_identity, seeded.snapshot_content_identity, + "the noop names the generation that was already sealed" + ); + assert_eq!( + scheduler.sealed_decode_count(), + decodes_before, + "keeping the sealed generation must not decode it again" + ); + assert_eq!( + RestoreFreshnessWitnessV1::load(store.path()) + .expect("rebound proof") + .generation_id, + generation_id.as_str(), + "the disk proof must name the sealed generation, not the predecessor" + ); + assert_eq!( + scheduler + .source_currency_witness_for(&generation_id, &metadata.snapshot().content_identity,) + .map(|witness| witness.generation_id), + Some(generation_id.clone()), + "the in-memory proof must admit the sealed generation" + ); + + fixture.edit( + "src/lib.rs", + "pub fn changed_after_predecessor_proof() -> u32 { 2 }\n", + ); + let refused = scheduler + .reconcile_retained_text_generation_with(&metadata, false) + .expect("changed source is a typed refusal, not an error"); + assert!( + refused.is_none(), + "moved bytes must not keep the sealed generation: {refused:?}" + ); + assert_eq!( + scheduler + .publication + .read_publication_pointer() + .expect("read pointer") + .expect("active pointer") + .generation_id, + generation_id.as_str(), + "refusing the moved bytes must not publish a substitute generation" + ); +} + +/// Clone backfill and the seal itself outlive the 30s admission window. Expiry +/// is a request to re-check the sealed digests, not a reason to drop the +/// generation those digests already name. A byte change after expiry still drops it. +#[test] +fn expired_proof_keeps_the_sealed_generation_until_bytes_move() { + let fixture = GitFixture::new(ALPHA_LIB_V1); + let store = TempDir::new().expect("store root"); + let mut scheduler = scheduler( + &fixture, + store.path().to_path_buf(), + Arc::new(SharedCodeIndexBytePoolV1::default()), + ); + let seeded = published(scheduler.reconcile_now().expect("seed retained generation")); + scheduler.expire_source_proof_for_test(); + assert_eq!( + scheduler + .currency_witness_for_sealed_snapshot( + &seeded.generation_id, + &seeded.snapshot_content_identity, + ) + .map(|witness| witness.generation_id), + Some(seeded.generation_id.clone()), + "an expired proof must keep the generation whose sealed bytes still match" + ); + + fixture.edit("src/lib.rs", "pub fn alpha() -> u32 { 9 }\n"); + scheduler.expire_source_proof_for_test(); + assert!( + scheduler + .currency_witness_for_sealed_snapshot( + &seeded.generation_id, + &seeded.snapshot_content_identity, + ) + .is_none(), + "an expired proof must drop the generation once its sealed bytes moved" + ); +} + /// A query freshness probe against a restored owner that no pass has verified /// yet must report "not current", the restart's first pass is still the /// remedy, without minting an observed source change: no overflow hint and no @@ -10377,3 +10648,66 @@ fn serving_swap_seats_a_generation_whose_publication_moved_while_it_activated() "neither refusing arm writes the serving slot" ); } + +/// Unchanged source bytes are not a reason to keep a generation the checkout +/// has committed past. An empty (or docs-only) commit moves HEAD without +/// touching one indexed byte, and `finish_retained_reconcile` rebuilds on +/// exactly that `source_revision` drift because branch-scoped reads resolve +/// generations by the commit they sealed. Accepting the sealed snapshot here +/// would pin the stale attribution for as long as the bytes hold still. +#[test] +fn a_moved_commit_refuses_the_sealed_generation_despite_identical_bytes() { + let fixture = GitFixture::new(ALPHA_LIB_V1); + let store = TempDir::new().expect("store root"); + let mut scheduler = scheduler( + &fixture, + store.path().to_path_buf(), + Arc::new(SharedCodeIndexBytePoolV1::default()), + ); + published(scheduler.reconcile_now().expect("seed retained generation")); + let metadata = scheduler + .servable_retained_text_generation() + .expect("publication store") + .expect("authenticated retained text generation") + .metadata() + .clone(); + let sealed_revision = metadata + .snapshot() + .source_revision + .clone() + .expect("a clean seed seals its commit"); + git( + fixture.path(), + &["commit", "-qm", "docs only", "--allow-empty"], + ); + let moved_head = + CommitId::new(git_stdout(fixture.path(), &["rev-parse", "HEAD"])).expect("moved HEAD"); + assert_ne!(sealed_revision, moved_head, "the fixture must move HEAD"); + + let refused = scheduler + .reconcile_retained_text_generation_with(&metadata, false) + .expect("graph-on retained reconcile"); + assert!( + refused.is_none(), + "a moved commit must not keep the generation sealed at {sealed_revision:?}: {refused:?}" + ); + + // The refusal is what hands the pass to the authoritative capture, and + // that capture is what re-attributes the generation to the new commit. + published( + scheduler + .reconcile_now() + .expect("rebuild at the moved commit"), + ); + assert_eq!( + scheduler + .servable_retained_text_generation() + .expect("publication store") + .expect("authenticated retained text generation") + .metadata() + .snapshot() + .source_revision, + Some(moved_head), + "the rebuilt generation must name the commit the checkout is on" + ); +} diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs index 306fa115a6..03ebce7c9e 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/serving.rs @@ -853,6 +853,95 @@ fn clone_successor_keeps_lexical_owners_ready_and_cas_replaces_v14() { assert_eq!(v16_revision, 16); } +/// Exact and lexical readiness is not the clone-successor copy. +/// +/// The publication advance that installs those owners used to call +/// `begin_clone_successor` before returning, and that call copies the whole +/// prior lexical artifact. The freshness receipt awaits that advance, so +/// status stayed non-current for the copy. The successor must still be +/// reported as backfill, and the next advance is what writes its staging file. +#[test] +fn lexical_readiness_leaves_the_clone_successor_uncopied() { + let fixture = GitFixture::new(&[( + "src/lib.rs", + "pub fn alpha() { one(); two(); three(); four(); five(); six(); seven(); eight(); nine(); ten(); }\n", + )]); + let store = TempDir::new().expect("store root"); + let mut scheduler = scheduler( + &fixture, + store.path().to_path_buf(), + Arc::new(SharedCodeIndexBytePoolV1::default()), + ); + published(scheduler.reconcile_now().expect("publish generation")); + let latest = scheduler.latest_complete().expect("latest generation"); + while !latest.query_owners_are_ready() { + latest.advance_text_serving(1).expect("advance V14 build"); + } + let tracedecay_contracts::code_index_freshness::CodeCloneIndexStatusV1::Backfilling { + observation, + } = latest.clone_index_status(false, None) + else { + panic!( + "a generation without clone fingerprints must report backfill once lexical owners serve, got {:?}", + latest.clone_index_status(false, None) + ); + }; + assert_eq!(observation.coverage.completed_source_pages, 0); + assert!( + observation.coverage.total_source_pages > 0, + "the pending successor must name the sealed page count it has not visited" + ); + // Status falls back to the published artifact's bytes when the successor + // has not created a staging file, so the bytes field cannot prove the + // copy stayed off this advance. The slot and the artifacts directory can. + assert!( + matches!( + &*latest.text_projection_build.lock_slot(), + super::super::CodeTextProjectionSlotV1::CloneSuccessorPending + ), + "owner readiness must leave the successor pending" + ); + let staging_names = |root: &std::path::Path| { + std::fs::read_dir(code_text_artifacts_root(root)) + .expect("artifacts root") + .map(|entry| entry.expect("artifact entry").file_name()) + .filter(|name| name.to_string_lossy().ends_with(".staging")) + .collect::>() + }; + assert!( + staging_names(store.path()).is_empty(), + "owner readiness copied the prior lexical artifact: {:?}", + staging_names(store.path()) + ); + + latest + .advance_text_serving(1) + .expect("the retained successor advance copies the prior artifact"); + assert!(latest.query_owners_are_ready()); + assert!( + !matches!( + &*latest.text_projection_build.lock_slot(), + super::super::CodeTextProjectionSlotV1::CloneSuccessorPending + ), + "the next advance must take the pending successor" + ); + + while latest.text_projection_needs_work() { + latest + .advance_text_serving(16) + .expect("finish clone successor"); + } + let revision: i64 = rusqlite::Connection::open(active_text_artifact_path(store.path())) + .expect("open finished artifact") + .query_row( + "SELECT format_revision FROM artifact_state WHERE singleton = 1", + [], + |row| row.get(0), + ) + .expect("read finished revision"); + assert_eq!(revision, 16); +} + #[test] fn clone_status_distinguishes_unavailable_backfill_partial_ready_and_stale() { let fixture = GitFixture::new(&[( diff --git a/crates/tracedecay-code-index/src/chunks.rs b/crates/tracedecay-code-index/src/chunks.rs index 8cdc6956be..08e9be3684 100644 --- a/crates/tracedecay-code-index/src/chunks.rs +++ b/crates/tracedecay-code-index/src/chunks.rs @@ -2102,8 +2102,14 @@ pub(crate) fn cross_file_reference_name_is_blocklisted(reference_name: &str) -> /// `Builder::default`). Keeps the intentional `` definition /// name while restoring same-file / seal recall for those calls. `None` when /// `path` is not a well-formed UFCS trait-impl method. +/// +/// A trait impl inside an inline module carries that module path +/// (`inner::::wide`); the alias keeps it, because same-file +/// resolution keys definitions by their whole file-relative name. pub(crate) fn rust_type_path_alias_for_trait_impl_method(path: &str) -> Option { - if !path.starts_with('<') { + let open = path.find('<')?; + let (module_prefix, path) = path.split_at(open); + if !(module_prefix.is_empty() || module_prefix.ends_with("::")) { return None; } let mut depth = 0_i32; @@ -2139,7 +2145,7 @@ pub(crate) fn rust_type_path_alias_for_trait_impl_method(path: &str) -> Option usize { 0 }\n", + " fn measure(&self) -> usize { self.len() }\n", + "}\n", + "trait Span {}\n", + "impl Span for Rows {\n", + " fn wide(&self) -> usize { self.len() }\n", + "}\n", + "\n", + "fn caller(items: Vec, rows: Rows) {\n", + " let foreign = make();\n", + " foreign.prepare(1);\n", + " items.push(1);\n", + " prepare(1);\n", + " push(1);\n", + " rows.len();\n", + "}\n", + "fn make() -> Vec { Vec::new() }\n", + ); + let file = validated_file("src/lib.rs", source.as_bytes()); + let batch = batch_for(&file, ParseOutcomeV1::Complete); + let artifacts = chunker() + .index_file(&file, &batch, &rust_descriptor(), &NeverCancelled) + .expect("indexing succeeds"); + let qualified = |occurrence: &SymbolOccurrenceId| { + artifacts + .symbols + .iter() + .find(|symbol| &symbol.occurrence == occurrence) + .map(|symbol| symbol.qualified_name.as_str()) + .unwrap_or("") + }; + let mut calls = artifacts + .edges + .iter() + .filter(|edge| edge.kind == RelationEdgeKindV1::Calls) + .map(|edge| { + ( + qualified(&edge.from_occurrence).to_owned(), + qualified(&edge.to_occurrence).to_owned(), + ) + }) + .collect::>(); + calls.sort(); + + assert_eq!( + calls, + vec![ + ( + "src/lib.rs::::wide".to_owned(), + "src/lib.rs::Rows::len".to_owned(), + ), + ( + "src/lib.rs::Rows::measure".to_owned(), + "src/lib.rs::Rows::len".to_owned(), + ), + ( + "src/lib.rs::caller".to_owned(), + "src/lib.rs::Rows::len".to_owned(), + ), + ( + "src/lib.rs::caller".to_owned(), + "src/lib.rs::make".to_owned(), + ), + ( + "src/lib.rs::caller".to_owned(), + "src/lib.rs::prepare".to_owned(), + ), + ( + "src/lib.rs::caller".to_owned(), + "src/lib.rs::push".to_owned() + ), + ], + "a bare receiver must not add a same-file caller; self and typed \ + bindings still bind: {calls:?}" + ); + } + + #[test] + fn self_call_inside_an_inline_module_binds_the_module_scoped_method() { + let source = concat!( + "mod inner {\n", + " pub struct Rows;\n", + " trait Wide { fn wide(&self) -> usize; }\n", + " impl Wide for Rows {\n", + " fn wide(&self) -> usize { 1 }\n", + " }\n", + " impl Rows {\n", + " fn len(&self) -> usize { 0 }\n", + " fn measure(&self) -> usize { self.len() + self.wide() }\n", + " }\n", + "}\n", + ); + let file = validated_file("src/lib.rs", source.as_bytes()); + let batch = batch_for(&file, ParseOutcomeV1::Complete); + let artifacts = chunker() + .index_file(&file, &batch, &rust_descriptor(), &NeverCancelled) + .expect("indexing succeeds"); + let qualified = |occurrence: &SymbolOccurrenceId| { + artifacts + .symbols + .iter() + .find(|symbol| &symbol.occurrence == occurrence) + .map(|symbol| symbol.qualified_name.as_str()) + .unwrap_or("") + }; + let mut calls = artifacts + .edges + .iter() + .filter(|edge| edge.kind == RelationEdgeKindV1::Calls) + .map(|edge| { + ( + qualified(&edge.from_occurrence).to_owned(), + qualified(&edge.to_occurrence).to_owned(), + ) + }) + .collect::>(); + calls.sort(); + + assert_eq!( + calls, + vec![ + ( + "src/lib.rs::inner::Rows::measure".to_owned(), + "src/lib.rs::inner::::wide".to_owned(), + ), + ( + "src/lib.rs::inner::Rows::measure".to_owned(), + "src/lib.rs::inner::Rows::len".to_owned(), + ), + ], + "`self` must name the module-scoped receiver type so same-file \ + resolution still binds: {calls:?}" + ); + let retained_calls = artifacts + .unresolved_references + .iter() + .filter(|reference| reference.kind == RelationEdgeKindV1::Calls) + .collect::>(); + assert!( + retained_calls.is_empty(), + "a same-file self call must not be retained as cross-file: \ + {retained_calls:?}" + ); + } + #[test] fn rust_type_path_alias_parses_ufcs_trait_impl_methods() { assert_eq!( diff --git a/crates/tracedecay-code-index/src/extract.rs b/crates/tracedecay-code-index/src/extract.rs index 85eb041ceb..cd4f39eb1a 100644 --- a/crates/tracedecay-code-index/src/extract.rs +++ b/crates/tracedecay-code-index/src/extract.rs @@ -776,12 +776,13 @@ mod tests { // extractor.rust.v8 records restricted `pub` re-export scope as a // typed value and no longer fabricates receiver types for method // initializers; v9 adds the clone-body token bound and v10 the byte - // bound. The revision is part of the batch identity, so the pinned - // digest moves with it. - assert_eq!(descriptor.extractor_revision.as_str(), "extractor.rust.v10"); + // bound; v11 drops the bare method name of a dotted call and types + // `self` from the enclosing impl or trait. The revision is part of the + // batch identity, so the pinned digest moves with it. + assert_eq!(descriptor.extractor_revision.as_str(), "extractor.rust.v11"); assert_eq!( extraction.batch().rows_digest.as_str(), - "sha256:2e1ebb8fbd7b438059eda5da2db9c8db9707f219484a28e82066caf527b038c6" + "sha256:e92b7ad8f93e3576c70adafb0690d064bd996c207a4ecd7b855c96e3ad3959ad" ); } diff --git a/crates/tracedecay-code-index/src/languages.rs b/crates/tracedecay-code-index/src/languages.rs index 9f3a01714c..5d7ea779ed 100644 --- a/crates/tracedecay-code-index/src/languages.rs +++ b/crates/tracedecay-code-index/src/languages.rs @@ -210,10 +210,13 @@ impl StaticLanguageRegistry { // the owning type. Every language moved one revision when clone-body // eligibility gained its token bound and again when it gained the // pre-tokenization byte bound: one multi-megabyte literal is only - // a few tokens but still cannot fit a text-artifact page. Only - // re-extraction removes the poisoned record. + // a few tokens but still cannot fit a text-artifact page. Rust v11 + // stopped emitting the bare method name of a dotted call, so an + // unrelated same-file callable sharing that name is no longer a + // caller, and types `self` through the enclosing impl or trait. + // Only re-extraction removes the poisoned record. let extractor_revision = if language == "rust" { - 10 + 11 } else if matches!(language.as_str(), "typescript" | "protobuf" | "sql") { 6 } else { @@ -418,7 +421,7 @@ mod tests { assert!(rust.stable_member_spans); assert!(rust.capabilities.extraction); assert_eq!(rust.root_markers, vec!["Cargo.toml".to_owned()]); - assert_eq!(rust.extractor_revision.as_str(), "extractor.rust.v10"); + assert_eq!(rust.extractor_revision.as_str(), "extractor.rust.v11"); assert_eq!( registry diff --git a/crates/tracedecay-code-index/src/production/worker_tests.rs b/crates/tracedecay-code-index/src/production/worker_tests.rs index cda982bd16..7531f94588 100644 --- a/crates/tracedecay-code-index/src/production/worker_tests.rs +++ b/crates/tracedecay-code-index/src/production/worker_tests.rs @@ -357,7 +357,7 @@ fn extractor_revision_change_reextracts_before_validating_retained_import_rows() assert_eq!( rebuilt.files[0].extraction.extractor_revision.as_str(), - "extractor.rust.v10" + "extractor.rust.v11" ); assert_ne!( rebuilt.files[0].extraction.parser_import_rows_digest, @@ -425,7 +425,7 @@ fn physical_artifact_reuse_rejects_a_stale_extractor_revision() { assert_eq!( rebuilt.files[0].extraction.extractor_revision.as_str(), - "extractor.rust.v10" + "extractor.rust.v11" ); assert!( rebuilt.files[0] diff --git a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs index 89c3b65fb3..6e7d5f7894 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs @@ -3244,22 +3244,22 @@ fn partitioned_codec_fixture() -> ( } const PARTITIONED_FORMAT_STATE_DIGEST: &str = - "sha256:8d84348830efc4452a078cfac1cc78e0ed44112a37f1025bd6f4f4bc152fe196"; + "sha256:4f78e1a1b0a4ea366f748e4699d4c28d9913782912809bbe2afafba4eac25268"; const PARTITIONED_FORMAT_SEGMENTS: &[(&str, u64)] = &[ ( - "sha256:e50d2733b5f594d79fdccc3e44b5d30d5efb66d14805b5fe0c67d5ceb0a1d66f", + "sha256:7cd13a44df02cc2dc2e13d5a867aaab5410594eaa398fe535ed59450ddc63b36", 11_071, ), ( - "sha256:1095d61bb8bbbf6637f85ca957a510d221aaba7923af8e60b0f3eef07042e6ff", + "sha256:64b5c4d5c08f363d66c1dc3922fb61df7c8df0b022dab6b72507e61e6e1bf403", 5_171, ), ( - "sha256:9921ca7da5c489307887ab570a5e8d5a7cebf192b9b6664c487e9a327943e396", + "sha256:782d1321bdc39aac9efcff85a44f8d48da66e24212d28150020cb04309d23880", 6_279, ), ( - "sha256:52b5707b5312bcb1e29849372b0dbb882205b3289b345643620a35c1d260c246", + "sha256:3a307f49e46059b54a86dac6921287afbe0b25cadc581840b5143ce2bd5a04d1", 6_837, ), ]; diff --git a/crates/tracedecay-contracts/src/lib.rs b/crates/tracedecay-contracts/src/lib.rs index f3049cc9b8..7f91f67774 100644 --- a/crates/tracedecay-contracts/src/lib.rs +++ b/crates/tracedecay-contracts/src/lib.rs @@ -344,10 +344,11 @@ pub use result::{ EvidenceScore, EvidenceScoreKind, EvidenceScoreValue, FreshnessState, IdempotencyKey, LegalAction, Omission, OmissionReason, OpaqueCursor, OperationBudgetUsage, OperationReceipt, OperationTermination, PageCursor, PageState, PolicyDecisionRef, PreviewId, PreviewResult, - ProblemOwningLayer, ProblemTerminality, ReconciliationState, ResultContractRef, ResumeToken, - RetrievalEvidence, RetrieverContribution, RetrieverContributionState, RetryDirective, - RetryScope, SafeDiagnostic, ScoreId, StreamEvent, StreamEventKind, StreamFrontier, StreamGap, - StreamTermination, StreamValidationError, TemporalState, validate_stream, + ProblemOwningLayer, ProblemTerminality, RUNTIME_MOUNTING_REASON_CODE, ReconciliationState, + ResultContractRef, ResumeToken, RetrievalEvidence, RetrieverContribution, + RetrieverContributionState, RetryDirective, RetryScope, SafeDiagnostic, ScoreId, StreamEvent, + StreamEventKind, StreamFrontier, StreamGap, StreamTermination, StreamValidationError, + TemporalState, validate_stream, }; pub use retained_receipts::{ PreparedRetainedEffect, authority_receipt, effective_memory_deadline, evidence_outcome, diff --git a/crates/tracedecay-contracts/src/result/envelope.rs b/crates/tracedecay-contracts/src/result/envelope.rs index 1499f09160..9377fc40a8 100644 --- a/crates/tracedecay-contracts/src/result/envelope.rs +++ b/crates/tracedecay-contracts/src/result/envelope.rs @@ -14,7 +14,7 @@ use super::{ ApplicationExecutionFailureClassV1, ApplicationProblem, ApplicationProblemKind, ApplicationUnavailableClassV1, CancellationStage, EffectReceipt, EffectResult, EvidenceCoverage, EvidencePacket, LegalAction, PreviewResult, ProblemOwningLayer, - ProblemTerminality, RetryDirective, RetryScope, SafeDiagnostic, + ProblemTerminality, RUNTIME_MOUNTING_REASON_CODE, RetryDirective, RetryScope, SafeDiagnostic, }; pub const APPLICATION_PROBLEM_REVISION: u32 = 1; @@ -591,11 +591,14 @@ impl ApplicationProblemRecord { self.terminality == ProblemTerminality::AdmittedTerminal } - /// The delay this problem directs before the same request may be sent - /// again, when it is a retryable pre-admission state such as a warming or - /// still-mounting authority. Admitted terminals and every other retry - /// directive answer `None`: nothing about the request should be repeated - /// on a timer. + /// The delay this problem's retry directive names, when an agent may send + /// the same request again. Admitted terminals and every other retry + /// directive answer `None`. + /// + /// This is not a transport instruction to loop. A retained authority that + /// is unavailable still carries `after_delay` so the caller can choose to + /// retry; only [`owner_mount_resend_delay`] tells the one-shot client to + /// re-send on its own. pub fn pre_admission_retry_delay(&self) -> Option { (self.retryable && self.retry == RetryDirective::AfterDelay && self.is_pre_admission()) .then_some(self.retry_after_millis) @@ -603,6 +606,20 @@ impl ApplicationProblemRecord { .map(Duration::from_millis) } + /// Delay before the one-shot client re-sends this completed result, or + /// `None` when the result is the answer. + /// + /// Classification keys on [`RUNTIME_MOUNTING_REASON_CODE`]. A publication + /// window that is still registering its owner changes if the same request + /// is sent again. Every other completed problem is returned on the first + /// observation, even when its directive is `after_delay`. + pub fn owner_mount_resend_delay(&self) -> Option { + if self.code != RUNTIME_MOUNTING_REASON_CODE { + return None; + } + self.pre_admission_retry_delay() + } + pub fn source(&self) -> &ApplicationProblem { &self.source } @@ -1002,4 +1019,40 @@ mod tests { serde_json::json!(["contract", "request_id", "problem"]) ); } + + fn retry_directed_record(code: &str, delay_millis: u64) -> ApplicationProblemRecord { + let envelope = ApplicationProblemEnvelope::new( + ResultContractRef::new( + SchemaId::new("schema.test.retry-directed.result").expect("schema id"), + 1, + ) + .expect("result contract"), + RequestId::new("request.test.retry-directed").expect("request id"), + ApplicationProblem::unavailable( + SafeDiagnostic::new(code, "The authority named by this code is not ready") + .expect("diagnostic"), + ), + ) + .expect("retry-directed envelope") + .with_retry_after_millis(Some(delay_millis)) + .expect("retry delay"); + *envelope.problem + } + + #[test] + fn only_a_mounting_refusal_is_resent_by_the_one_shot_client() { + let mounting = retry_directed_record(RUNTIME_MOUNTING_REASON_CODE, 40); + let answered = retry_directed_record("application.retained.authority-unavailable", 40); + + assert_eq!( + mounting.owner_mount_resend_delay(), + Some(Duration::from_millis(40)) + ); + assert_eq!( + answered.pre_admission_retry_delay(), + Some(Duration::from_millis(40)), + "the caller-facing directive still names the delay" + ); + assert_eq!(answered.owner_mount_resend_delay(), None); + } } diff --git a/crates/tracedecay-contracts/src/result/mod.rs b/crates/tracedecay-contracts/src/result/mod.rs index abbdb68e36..d59a83277c 100644 --- a/crates/tracedecay-contracts/src/result/mod.rs +++ b/crates/tracedecay-contracts/src/result/mod.rs @@ -19,7 +19,7 @@ pub use evidence::{ pub use problem::{ ApplicationExecutionFailureClassV1, ApplicationProblem, ApplicationProblemKind, ApplicationUnavailableClassV1, LegalAction, ProblemOwningLayer, ProblemTerminality, - RetryDirective, RetryScope, SafeDiagnostic, + RUNTIME_MOUNTING_REASON_CODE, RetryDirective, RetryScope, SafeDiagnostic, }; pub use receipt::{ CancellationObservation, CancellationStage, EffectId, EffectReceipt, EffectResult, diff --git a/crates/tracedecay-contracts/src/result/problem.rs b/crates/tracedecay-contracts/src/result/problem.rs index bc0ca13646..013d638b55 100644 --- a/crates/tracedecay-contracts/src/result/problem.rs +++ b/crates/tracedecay-contracts/src/result/problem.rs @@ -5,6 +5,15 @@ use tracedecay_domain::errors::TraceDecayError; use super::{CancellationStage, EffectReceipt, EffectTermination}; use crate::error::ApplicationContractError; +/// Diagnostic code for an admitted route whose owner is still registering +/// behind the core publication. +/// +/// The one-shot client re-sends the same request only while this code is the +/// problem: the next observation can be the owner's answer. Every other +/// completed problem, including a retryable authority unavailable, is already +/// the daemon's answer and must not be reconnected. +pub const RUNTIME_MOUNTING_REASON_CODE: &str = "application.runtime.mounting"; + /// Safe adapter-independent retry instruction. Adapters preserve it verbatim. #[derive( Clone, Copy, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq, PartialOrd, Ord, Hash, diff --git a/crates/tracedecay-contracts/src/result/problem/tests.rs b/crates/tracedecay-contracts/src/result/problem/tests.rs index 69ba848bee..a4cfcb21b8 100644 --- a/crates/tracedecay-contracts/src/result/problem/tests.rs +++ b/crates/tracedecay-contracts/src/result/problem/tests.rs @@ -157,7 +157,7 @@ fn application_problem_converts_to_typed_trace_decay_error() { let warming = ApplicationProblem::unavailable( SafeDiagnostic::new( - "application.surface.unavailable", + super::RUNTIME_MOUNTING_REASON_CODE, "The project runtime for this operation is still mounting", ) .expect("fixture diagnostic is valid"), @@ -166,7 +166,7 @@ fn application_problem_converts_to_typed_trace_decay_error() { let (reason_code, retryable, _) = warming_error .project_route_context() .expect("warming stays a project-route error"); - assert_eq!(reason_code, "application.surface.unavailable"); + assert_eq!(reason_code, super::RUNTIME_MOUNTING_REASON_CODE); assert!(retryable); } diff --git a/crates/tracedecay-daemon-service/src/invocation/tests/project_admission_tests.rs b/crates/tracedecay-daemon-service/src/invocation/tests/project_admission_tests.rs index 4c83b5a136..75a01fe439 100644 --- a/crates/tracedecay-daemon-service/src/invocation/tests/project_admission_tests.rs +++ b/crates/tracedecay-daemon-service/src/invocation/tests/project_admission_tests.rs @@ -137,7 +137,7 @@ async fn admitted_storage_status_stays_retryable_while_owners_are_warming() { problem .diagnostic() .map(|diagnostic| diagnostic.code.as_str()), - Some("application.surface.unavailable") + Some(tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE) ); } @@ -227,7 +227,7 @@ async fn storage_status_admits_an_owner_registered_under_a_windows_verbatim_root problem .diagnostic() .map(|diagnostic| diagnostic.code.as_str()), - Some("application.surface.unavailable") + Some(tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE) ); } @@ -300,7 +300,10 @@ async fn retained_request_stays_retryable_while_owners_are_warming() { let diagnostic = problem .diagnostic() .expect("a mounting retained owner carries a diagnostic"); - assert_eq!(diagnostic.code, "application.surface.unavailable"); + assert_eq!( + diagnostic.code, + tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE + ); assert!( !diagnostic .message diff --git a/crates/tracedecay-daemon-service/src/invocation/work.rs b/crates/tracedecay-daemon-service/src/invocation/work.rs index 67f7b691f8..7da13757b9 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work.rs @@ -47,7 +47,7 @@ pub(super) fn runtime_mounting_problem(request_id: String) -> DaemonInvocationRe application_problem( request_id, ApplicationProblem::unavailable(SafeDiagnostic { - code: "application.surface.unavailable".to_owned(), + code: tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE.to_owned(), message: "The project runtime for this operation is still mounting".to_owned(), }), ) diff --git a/crates/tracedecay-daemon-service/src/profile_host_admission_replay.rs b/crates/tracedecay-daemon-service/src/profile_host_admission_replay.rs index f8d79e2cd4..255e0622ef 100644 --- a/crates/tracedecay-daemon-service/src/profile_host_admission_replay.rs +++ b/crates/tracedecay-daemon-service/src/profile_host_admission_replay.rs @@ -841,6 +841,10 @@ impl ProfileHostAdmissionReplayWorker { // Non-retryable failure: stop until the next explicit kick. break; } + ReplayPassDecision::TerminalNoop => { + consecutive_retryable = 0; + break; + } ReplayPassDecision::Requeue => { consecutive_retryable = 0; } @@ -1391,6 +1395,43 @@ mod tests { registry.shutdown().await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn not_applicable_pending_record_does_not_back_off() { + let temp = tempfile::TempDir::new().unwrap(); + let profile_root = temp.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + let db_path = tracedecay_sessions::runtime::user_sessions_db_path(&profile_root); + let (runtime, _) = + tracedecay_host_admission::HostAdmissionRuntime::open_for_database(&db_path).unwrap(); + let broker = Arc::new(tracedecay_host_admission::HostAdmissionBroker::new(runtime)); + broker.admit("test:pending", b"pending").await.unwrap(); + let registry = ProfileHostAdmissionReplayRegistry::default(); + let pass_override = Arc::new(|| { + Box::pin(async { HostAdmissionOutcome::not_applicable("code_index_not_applicable") }) + as std::pin::Pin + Send>> + }); + + registry + .ensure_with_pass_override(&db_path, &profile_root, &broker, pass_override) + .await; + tokio::time::timeout(Duration::from_secs(1), async { + while registry.pass_count(&db_path).await == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("profile replay must attempt the not-applicable record"); + tokio::time::sleep(Duration::from_millis(150)).await; + let passes = registry.pass_count(&db_path).await; + assert!(passes >= 1); + assert_eq!(registry.backoff_count(&db_path).await, 0); + tokio::time::sleep(Duration::from_millis(150)).await; + + assert_eq!(registry.pass_count(&db_path).await, passes); + assert_eq!(registry.backoff_count(&db_path).await, 0); + registry.shutdown().await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn shutdown_cancels_and_joins_an_in_flight_pass() { let temp = tempfile::TempDir::new().unwrap(); diff --git a/crates/tracedecay-domain/src/observation.rs b/crates/tracedecay-domain/src/observation.rs index f9fca1fc0e..954855587e 100644 --- a/crates/tracedecay-domain/src/observation.rs +++ b/crates/tracedecay-domain/src/observation.rs @@ -878,6 +878,19 @@ impl ObservationSourceCursorV1 { } Ok(self.byte_offset.cmp(&other.byte_offset)) } + + /// Whether this cursor already owns `frontier` on the same ordering authority. + /// + /// Progress is the position. Resume fingerprints are checkpoints, not + /// coverage, so two owners of the same bytes can disagree there without + /// either being behind the frontier. + #[must_use] + pub fn reached(&self, frontier: &Self) -> bool { + matches!( + self.checked_cmp(frontier), + Ok(Ordering::Equal | Ordering::Greater) + ) + } } pub const CANONICAL_OBSERVATION_ENVELOPE_VERSION_V1: u16 = 1; diff --git a/crates/tracedecay-global-db/src/observation_adapter.rs b/crates/tracedecay-global-db/src/observation_adapter.rs index 571f4c6fbc..965107ef73 100644 --- a/crates/tracedecay-global-db/src/observation_adapter.rs +++ b/crates/tracedecay-global-db/src/observation_adapter.rs @@ -1485,8 +1485,15 @@ impl ObservationStore for GlobalDbObservationStore { advance.next_cursor().source(), advance.next_cursor().scope(), )?; - let existed_at_next = actual_cursor.as_ref() == Some(advance.next_cursor()); - if !existed_at_next && actual_cursor.as_ref() != advance.expected_cursor() { + // One owner per frontier. A cursor that already reached `next` has + // recorded the range; a second reason must not become a permanent + // collision that both ingest owners then warn on forever. The + // command still goes to the writer so the current authority epoch + // receipts the replay; only its outcome is reported as a duplicate. + let reached_frontier = actual_cursor + .as_ref() + .is_some_and(|cursor| cursor.reached(advance.next_cursor())); + if !reached_frontier && actual_cursor.as_ref() != advance.expected_cursor() { return Err(ObservationStoreError::CursorConflict { expected: Box::new(advance.expected_cursor().cloned()), actual: Box::new(actual_cursor), @@ -1498,6 +1505,7 @@ impl ObservationStore for GlobalDbObservationStore { "coverage": advance.coverage(), }); let key = format!("cursor.{}", canonical_runtime_digest(&identity)?); + let next_cursor = advance.next_cursor().clone(); let payload = RepositoryWritePayloadV1::ObservationCursorAdvance(Box::new(advance)); let (command_bytes, command_digest) = canonical_json_bytes_and_sha256( &runtime_command_value(&payload)?, @@ -1517,7 +1525,7 @@ impl ObservationStore for GlobalDbObservationStore { match outcome? { RuntimeSubmitOutcomeV1::Committed { .. } | RuntimeSubmitOutcomeV1::CommittedAfterCancellation { .. } - if existed_at_next => + if reached_frontier => { Ok(CursorAdvanceOutcome::ExactDuplicate) } @@ -1526,19 +1534,21 @@ impl ObservationStore for GlobalDbObservationStore { Ok(CursorAdvanceOutcome::Committed) } RuntimeSubmitOutcomeV1::ExactReplay { .. } => Ok(CursorAdvanceOutcome::ExactDuplicate), - // The idempotency key covers the advanced coverage, not the whole - // command, so a re-scan of already-admitted history reuses the key - // with different bytes (a fresh `expected_cursor` or resume - // checkpoint) and the writer reports a conflict against the earlier - // committed receipt. When the durable cursor is already exactly - // `next_cursor`, that earlier commit is this advance: the coverage - // is applied and the replay is a duplicate. Only a conflict that - // left the cursor somewhere else is an unresolved collision. - RuntimeSubmitOutcomeV1::IdempotencyConflict { .. } if existed_at_next => { - Ok(CursorAdvanceOutcome::ExactDuplicate) - } + // The other owner committed this coverage key between the + // pre-check and the writer lookup. If the frontier moved, that + // owner already holds the range; the different command digest is + // not a durable collision. RuntimeSubmitOutcomeV1::IdempotencyConflict { .. } => { - Err(ObservationStoreError::CursorAdvanceCollision) + let raced = + read_runtime_source_cursor(runtime, next_cursor.source(), next_cursor.scope())?; + if raced + .as_ref() + .is_some_and(|cursor| cursor.reached(&next_cursor)) + { + Ok(CursorAdvanceOutcome::ExactDuplicate) + } else { + Err(ObservationStoreError::CursorAdvanceCollision) + } } other => Err(runtime_storage_error( "advance observation source cursor", diff --git a/crates/tracedecay-global-db/src/observation_collision_tests.rs b/crates/tracedecay-global-db/src/observation_collision_tests.rs index 4fd33d97cc..d1a8c76135 100644 --- a/crates/tracedecay-global-db/src/observation_collision_tests.rs +++ b/crates/tracedecay-global-db/src/observation_collision_tests.rs @@ -2975,6 +2975,72 @@ async fn failed_coverage_advance_leaves_no_visible_refusal_marker() { ); } +#[tokio::test] +async fn covered_frontier_keeps_the_first_reason_when_a_second_owner_advances() { + let tmp = TempDir::new().unwrap(); + let runtime = HostAdmissionTestRuntimeV1::profile(tmp.path()) + .await + .unwrap(); + let store = runtime + .observation_store(HostAdmissionScope::Profile) + .unwrap(); + let session_id = SessionId::new("session.cursor-owned-frontier").unwrap(); + let (observation, _) = collision_candidate( + &session_id, + "record.cursor-owned-frontier", + 1, + "owned frontier fixture", + "receipt.cursor-owned-frontier", + None, + ); + let advance = ObservationCursorAdvance::for_ordering( + observation.source().clone(), + observation.scope().clone(), + observation.identity().generation(), + observation.identity().ordering_domain(), + None, + observation.identity().position(), + ObservationCoverageReason::OutOfScope, + ) + .unwrap(); + seed_cursor_replay( + &runtime, + &advance, + Some(ObservationCoverageReason::BlankFrame), + ) + .await; + + assert_eq!( + store.advance_source_cursor(advance.clone()).await.unwrap(), + CursorAdvanceOutcome::ExactDuplicate + ); + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .unwrap(); + let snapshot = database.read_snapshot().await.unwrap(); + let mut rows = snapshot + .query( + "SELECT reason, COUNT(*) FROM source_cursor_advances GROUP BY reason", + (), + ) + .await + .unwrap(); + let row = rows.next().await.unwrap().expect("owned ledger row"); + assert_eq!(row.get::(0).unwrap(), "blank_frame"); + assert_eq!(row.get::(1).unwrap(), 1); + assert!(rows.next().await.unwrap().is_none()); + drop(rows); + assert_eq!( + store + .get_source_cursor(observation.source(), observation.scope()) + .await + .unwrap() + .as_ref() + .map(ObservationSourceCursorV1::position), + Some(advance.next_cursor().position()) + ); +} + #[tokio::test] async fn runtime_cursor_replay_preserves_structured_ledger_disagreement() { let tmp = TempDir::new().unwrap(); @@ -3009,6 +3075,18 @@ async fn runtime_cursor_replay_preserves_structured_ledger_disagreement() { Some(ObservationCoverageReason::BlankFrame), ) .await; + // The seeded cursor already stands at `next`. Pull it back so this + // advance is the write that would move the frontier, where a stored + // reason still disagrees. + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .unwrap(); + let transaction = database.begin_write_transaction().await.unwrap(); + transaction + .execute("DELETE FROM source_cursors", ()) + .await + .unwrap(); + transaction.commit().await.unwrap(); let error = store .advance_source_cursor(advance.clone()) @@ -3039,7 +3117,7 @@ async fn runtime_cursor_replay_preserves_structured_ledger_disagreement() { } #[tokio::test] -async fn runtime_cursor_replay_without_a_ledger_row_keeps_generic_collision_semantics() { +async fn covered_cursor_without_a_ledger_row_is_already_owned() { let tmp = TempDir::new().unwrap(); let runtime = HostAdmissionTestRuntimeV1::profile(tmp.path()) .await @@ -3068,10 +3146,123 @@ async fn runtime_cursor_replay_without_a_ledger_row_keeps_generic_collision_sema .unwrap(); seed_cursor_replay(&runtime, &advance, None).await; - assert!(matches!( - store.advance_source_cursor(advance).await.unwrap_err(), - ObservationStoreError::CursorAdvanceCollision - )); + assert_eq!( + store.advance_source_cursor(advance).await.unwrap(), + CursorAdvanceOutcome::ExactDuplicate + ); +} + +#[tokio::test] +async fn concurrent_cursor_owners_with_different_reasons_share_one_frontier() { + let tmp = TempDir::new().unwrap(); + let runtime = HostAdmissionTestRuntimeV1::profile(tmp.path()) + .await + .unwrap(); + let store = runtime + .observation_store(HostAdmissionScope::Profile) + .unwrap(); + let session_id = SessionId::new("session.cursor-concurrent-owners").unwrap(); + let (observation, _) = collision_candidate( + &session_id, + "record.cursor-concurrent-owners", + 1, + "concurrent owners fixture", + "receipt.cursor-concurrent-owners", + None, + ); + let blank = ObservationCursorAdvance::for_ordering( + observation.source().clone(), + observation.scope().clone(), + observation.identity().generation(), + observation.identity().ordering_domain(), + None, + observation.identity().position(), + ObservationCoverageReason::BlankFrame, + ) + .unwrap(); + let out_of_scope = ObservationCursorAdvance::for_ordering( + observation.source().clone(), + observation.scope().clone(), + observation.identity().generation(), + observation.identity().ordering_domain(), + None, + observation.identity().position(), + ObservationCoverageReason::OutOfScope, + ) + .unwrap(); + // Hold the writer so both owners pass the pre-check against the empty + // frontier and only then race the same coverage key. A short-circuit + // after one has already committed would not exercise the conflict path. + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .unwrap(); + let gate = database.begin_write_transaction().await.unwrap(); + let left_store = store.clone(); + let right_store = store.clone(); + let mut left_task = tokio::spawn(async move { left_store.advance_source_cursor(blank).await }); + let mut right_task = + tokio::spawn(async move { right_store.advance_source_cursor(out_of_scope).await }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(200), &mut left_task) + .await + .is_err(), + "the blank-frame owner must wait behind the held writer" + ); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(200), &mut right_task) + .await + .is_err(), + "the out-of-scope owner must wait behind the held writer" + ); + gate.rollback().await.unwrap(); + let left = left_task + .await + .unwrap() + .expect("blank-frame owner must not collide"); + let right = right_task + .await + .unwrap() + .expect("out-of-scope owner must not collide"); + let outcomes = [left, right]; + assert_eq!( + outcomes + .iter() + .filter(|outcome| **outcome == CursorAdvanceOutcome::Committed) + .count(), + 1, + "exactly one owner commits the frontier, got {outcomes:?}" + ); + assert_eq!( + outcomes + .iter() + .filter(|outcome| **outcome == CursorAdvanceOutcome::ExactDuplicate) + .count(), + 1, + "the other owner observes the owned frontier, got {outcomes:?}" + ); + assert_eq!(table_count(&runtime, "source_cursor_advances").await, 1); + assert_eq!(table_count(&runtime, "source_cursors").await, 1); + let cursor = only_source_cursor(&runtime).await; + assert_eq!(cursor.position(), observation.identity().position().end()); + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .unwrap(); + let snapshot = database.read_snapshot().await.unwrap(); + let mut rows = snapshot + .query("SELECT reason FROM source_cursor_advances", ()) + .await + .unwrap(); + let reason = rows + .next() + .await + .unwrap() + .expect("one ledger reason") + .get::(0) + .unwrap(); + assert!( + reason == "blank_frame" || reason == "out_of_scope", + "the retained reason must be one of the two owners, got {reason}" + ); } /// Overwrites the durable cursor for one source, the shape a retained-history @@ -3228,14 +3419,25 @@ async fn runtime_cursor_replay_preserves_storage_failure() { .unwrap(); seed_cursor_replay(&runtime, &advance, None).await; - assert!(matches!( - store - .advance_source_cursor(advance.clone()) - .await - .unwrap_err(), - ObservationStoreError::CursorAdvanceCollision - )); + assert_eq!( + store.advance_source_cursor(advance.clone()).await.unwrap(), + CursorAdvanceOutcome::ExactDuplicate + ); + let advance = ObservationCursorAdvance::for_ordering( + ObservationSourceIdentityV1::for_provider( + ProviderId::new(COLLISION_PROVIDER).unwrap(), + SessionId::new("session.cursor-runtime-storage-uncovered").unwrap(), + ) + .unwrap(), + ObservationScopeV1::Profile, + observation.identity().generation(), + observation.identity().ordering_domain(), + None, + observation.identity().position(), + ObservationCoverageReason::OutOfScope, + ) + .unwrap(); let database = runtime .registered_database(HostAdmissionScope::Profile) .unwrap(); diff --git a/crates/tracedecay-global-db/src/observation_projection.rs b/crates/tracedecay-global-db/src/observation_projection.rs index 3b609ffb82..2b58d746cd 100644 --- a/crates/tracedecay-global-db/src/observation_projection.rs +++ b/crates/tracedecay-global-db/src/observation_projection.rs @@ -29,6 +29,6 @@ pub(crate) use state::rearm_queued_projection_retries; #[cfg(test)] pub(super) use state::verify_projection_rows; pub(super) use state::{ - ProjectionOutputAuthority, ProjectionRowsBatch, read_output_authorities, + ProjectionOutputAuthority, ProjectionRowsBatch, load_verified_session, read_output_authorities, read_projection_rows_batch, resolve_output_projection, verify_projection_rows_from_records, }; diff --git a/crates/tracedecay-global-db/src/observation_projection/apply.rs b/crates/tracedecay-global-db/src/observation_projection/apply.rs index a9ad8c7ce1..5bcf7aabf8 100644 --- a/crates/tracedecay-global-db/src/observation_projection/apply.rs +++ b/crates/tracedecay-global-db/src/observation_projection/apply.rs @@ -633,6 +633,34 @@ pub(super) async fn apply_session( } } +/// Aligns a provenance-owned raw twin onto the projection's session before +/// the content upsert. +/// +/// The ingest upsert refuses a row whose `session_id` differs, so a drifted +/// twin blocks the rewrite that uniquely owned current provenance authorizes. +/// `(provider, message_id)` is that ownership key; `session_id` is a field of +/// the twin, not a second owner. Callers reach this only after that ownership +/// is already proven (an existing projected message, or released-rendering +/// convergence). A first insert of an unowned identity must not adopt a +/// foreign twin and does not call this. +async fn adopt_owned_projection_raw_session( + conn: &impl Executor, + message: &SessionMessageRecord, +) -> ProjectionStoreResult<()> { + conn.execute( + "UPDATE lcm_raw_messages SET session_id = ?3 + WHERE provider = ?1 AND message_id = ?2 AND session_id <> ?3", + params![ + message.provider.as_str(), + message.message_id.as_str(), + message.session_id.as_str(), + ], + ) + .await + .map(|_| ()) + .map_err(|error| storage("adopt projection raw session", error)) +} + /// Writes the projection-derived raw row through the canonical LCM raw /// authority so it carries the content-bound sanitization receipt that /// hydration requires; a receipt-less raw row is unreadable, not raw storage. @@ -820,13 +848,14 @@ pub(in super::super) enum ConvergedRendering { /// deterministic rendering, keeping the historical `message_created` flag the /// releases wrote. /// -/// Reached only from the authority audit, which has already proven the stored -/// provenance row is the digest of the output row this store holds, the -/// rendering a release wrote, rather than a row disagreeing with its own -/// output. The message row and its LCM raw twin are pure derivations of the -/// durable observation, so rewriting them loses nothing; the digest is -/// re-stamped last so an interrupted transaction leaves the released pairing -/// intact. +/// Reached only from the authority audit, which has already admitted the row +/// as a shipped rendering: provenance still carries the digest of the output +/// this store holds, or it carries this binary's digest while the mutable row +/// is still that shipped rendering. A row that matches neither is refused +/// before this write. The message row and its LCM raw twin are pure +/// derivations of the durable observation, so rewriting them loses nothing; +/// the digest is re-stamped last so an interrupted transaction leaves the +/// released pairing intact. /// /// When the LCM privacy sanitizer withholds this binary's rendering, that /// verdict *is* the current rendering: the output is retired to the disposition @@ -844,6 +873,7 @@ pub(in super::super) async fn converge_released_output_rendering( let message = projection.message(); supersede_projected_message(conn, message).await?; if message.provider != "hermes" { + adopt_owned_projection_raw_session(conn, message).await?; match upsert_projected_raw_message(conn, message).await { Ok(()) => {} Err(ProjectionStoreError::SanitizationRefused { @@ -1022,6 +1052,13 @@ async fn apply_rows( } }; if projected_message.provider != "hermes" && !preserve_protected_payload { + // Message-row presence is not projector ownership. An equal + // pre-existing row with no output state is retained without this + // projector ever having claimed the output, so its twin keeps the + // upsert's session guard and a disagreement stays a typed refusal. + if state.is_some_and(|state| state.projector_owned) { + adopt_owned_projection_raw_session(conn, projected_message).await?; + } upsert_projected_raw_message(conn, projected_message).await?; } Ok(transition == MessageTransition::Insert) diff --git a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs index 5f80655c15..2501f2712d 100644 --- a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs +++ b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs @@ -8,7 +8,7 @@ use tracedecay_lcm::retrieval_content::{ use tracedecay_runtime_core::db::engine::{Connection, TransactionBehavior}; use tracedecay_runtime_core::db::{ Database, - engine::{Executor, QueryExecutor, params}, + engine::{Executor, QueryExecutor, Row, params}, }; use tracedecay_store::{ ObservationProjection, PROVIDER_USAGE_PROJECTOR_VERSION, ProjectedObservation, @@ -43,6 +43,7 @@ const PROJECTION_RETRY_MAX_MICROS: i64 = 300_000_000; static NEVER_CANCELLED: AtomicBool = AtomicBool::new(false); const SESSION_JSON_COLUMN: &str = "session_json"; +const MERGED_SESSION_JSON_COLUMN: &str = "merged.value"; const MESSAGE_JSON_COLUMN: &str = "message_json"; const STAGED_MESSAGE_JSON_COLUMN: &str = "staged.message_json"; @@ -2193,76 +2194,167 @@ async fn retire_projection_predecessor_output_ownership( .map_err(|error| storage("retire predecessor projection provenance", error)) } -async fn activate_rebuild_sessions( +fn decode_overlapping_session(row: &Row) -> ProjectionStoreResult { + macro_rules! cell { + ($index:literal) => { + row.get($index) + .map_err(|error| storage("decode overlapping projection session", error))? + }; + ($index:literal, $ty:ty) => { + row.get::<$ty>($index) + .map_err(|error| storage("decode overlapping projection session", error))? + }; + } + Ok(SessionRecord { + provider: cell!(1), + session_id: cell!(2), + project_key: cell!(3), + project_path: cell!(4), + title: cell!(5), + started_at: cell!(6), + ended_at: cell!(7), + transcript_path: cell!(8), + metadata_json: cell!(9), + parent_session_id: cell!(10), + is_subagent: cell!(11, i64) != 0, + agent_id: cell!(12), + parent_tool_use_id: cell!(13), + }) +} + +/// Write every reconciled overlap in one set-based statement. Rebuild +/// activation owns the database writer, so a per-row `UPDATE` loop would hold +/// admission for as long as the history is large; the merge itself already ran +/// in Rust, so each column is taken verbatim from the merged row. +async fn write_reconciled_sessions( + conn: &impl Executor, + merged: &[SessionRecord], +) -> ProjectionStoreResult<()> { + if merged.is_empty() { + return Ok(()); + } + let rows = encode_json(&merged, "encode reconciled projection sessions")?; + let session_extracts = + json_extract_select_list(MERGED_SESSION_JSON_COLUMN, SESSION_JSON_FIELDS); + let assignments = SESSION_JSON_FIELDS + .iter() + .map(|field| format!("{field} = excluded.{field}")) + .collect::>() + .join(",\n "); + conn.execute( + &format!( + "INSERT INTO sessions ( + provider, session_id, project_key, project_path, title, started_at, ended_at, + transcript_path, metadata_json, parent_session_id, is_subagent, agent_id, + parent_tool_use_id + ) + SELECT {}, {}, + {session_extracts} + FROM json_each(?1) AS merged + -- `WHERE true` disambiguates the upsert clause from a join constraint. + WHERE true + ON CONFLICT(provider, session_id) DO UPDATE SET + {assignments}", + json_extract_expr(MERGED_SESSION_JSON_COLUMN, "provider"), + json_extract_expr(MERGED_SESSION_JSON_COLUMN, "session_id"), + ), + params![rows.as_str()], + ) + .await + .map(|_| ()) + .map_err(|error| storage("activate reconciled projection sessions", error)) +} + +/// Classify every staged session that already exists through +/// [`reconcile_session_rows_detailed`], the same authority live apply uses. +/// A parallel SQL predicate used to report those conflicts as message +/// `OutputCollision` values with `message_id = session:{id}`, which erased +/// the field and sent session conflicts down the message-skip path. +/// Paged by `(provider, session_id)` because the exact-SQL transport refuses a +/// result set past `MAX_QUERY_ROWS` (10_000 rows) or 64 MiB, and a rebuild +/// overlapping more history than that would fail activation instead of +/// reconciling it. Both the staged primary key and `sessions` are unique on +/// that pair, so one page's writes never move a later page's cursor. +async fn reconcile_overlapping_rebuild_sessions( conn: &impl Executor, generation: &str, ) -> ProjectionStoreResult<()> { - let mut conflicts = conn - .query( - "SELECT staged.provider, staged.session_id + let mut cursor: Option<(String, String)> = None; + loop { + let (after_provider, after_session) = match cursor.as_ref() { + Some((provider, session_id)) => (Some(provider.as_str()), Some(session_id.as_str())), + None => (None, None), + }; + let mut overlaps = conn + .query( + "SELECT staged.session_json, + active.provider, active.session_id, active.project_key, + active.project_path, active.title, active.started_at, + active.ended_at, active.transcript_path, active.metadata_json, + active.parent_session_id, active.is_subagent, active.agent_id, + active.parent_tool_use_id FROM observation_projection_rebuild_sessions AS staged JOIN sessions AS active ON active.provider = staged.provider AND active.session_id = staged.session_id WHERE staged.projector_version = ?1 AND staged.generation = ?2 - AND ( - (active.project_key <> json_extract(staged.session_json, '$.project_key') - AND active.project_key <> 'user' - AND json_extract(staged.session_json, '$.project_key') <> 'user') - OR (active.project_path <> json_extract(staged.session_json, '$.project_path') - AND active.project_path <> active.project_key - AND json_extract(staged.session_json, '$.project_path') - <> json_extract(staged.session_json, '$.project_key')) - OR (active.transcript_path IS NOT NULL - AND json_extract(staged.session_json, '$.transcript_path') IS NOT NULL - AND active.transcript_path IS NOT json_extract(staged.session_json, '$.transcript_path')) - OR (active.parent_session_id IS NOT NULL - AND json_extract(staged.session_json, '$.parent_session_id') IS NOT NULL - AND active.parent_session_id IS NOT json_extract(staged.session_json, '$.parent_session_id')) - OR (active.agent_id IS NOT NULL - AND json_extract(staged.session_json, '$.agent_id') IS NOT NULL - AND active.agent_id IS NOT json_extract(staged.session_json, '$.agent_id')) - OR (active.parent_tool_use_id IS NOT NULL - AND json_extract(staged.session_json, '$.parent_tool_use_id') IS NOT NULL - AND active.parent_tool_use_id IS NOT json_extract(staged.session_json, '$.parent_tool_use_id')) - OR (active.metadata_json IS NOT NULL - AND json_extract(staged.session_json, '$.metadata_json') IS NOT NULL - AND active.metadata_json IS NOT json_extract(staged.session_json, '$.metadata_json') - AND ( - json_valid(active.metadata_json) = 0 - OR json_valid(json_extract(staged.session_json, '$.metadata_json')) = 0 - OR json_type(active.metadata_json) <> 'object' - OR json_type(json_extract(staged.session_json, '$.metadata_json')) <> 'object' - OR EXISTS ( - SELECT 1 - FROM json_each(json_extract(staged.session_json, '$.metadata_json')) AS expected - JOIN json_each(active.metadata_json) AS actual USING (key) - WHERE expected.key NOT IN ('source', 'usage') - AND actual.value IS NOT expected.value - ) - )) - ) - LIMIT 1", - params![SESSION_MESSAGE_PROJECTOR_VERSION, generation], - ) - .await - .map_err(|error| storage("validate staged projection sessions", error))?; - if let Some(row) = conflicts - .next() - .await - .map_err(|error| storage("validate staged projection sessions", error))? - { - return Err(ProjectionStoreError::OutputCollision { - provider: row + AND (?3 IS NULL + OR staged.provider > ?3 + OR (staged.provider = ?3 AND staged.session_id > ?4)) + ORDER BY staged.provider, staged.session_id + LIMIT ?5", + params![ + SESSION_MESSAGE_PROJECTOR_VERSION, + generation, + after_provider, + after_session, + REBUILD_PAGE_SIZE + ], + ) + .await + .map_err(|error| storage("read overlapping projection sessions", error))?; + let mut updates = Vec::new(); + let mut scanned = 0_i64; + let mut last = None; + while let Some(row) = overlaps + .next() + .await + .map_err(|error| storage("read overlapping projection sessions", error))? + { + let staged_json: String = row .get(0) - .map_err(|error| storage("validate staged projection sessions", error))?, - message_id: format!( - "session:{}", - row.get::(1) - .map_err(|error| storage("validate staged projection sessions", error))? - ), - }); + .map_err(|error| storage("read overlapping projection sessions", error))?; + let staged: SessionRecord = + decode_json(&staged_json, "decode staged projection session")?; + let actual = decode_overlapping_session(&row)?; + scanned += 1; + last = Some((actual.provider.clone(), actual.session_id.clone())); + let expected = canonicalize_session_project_paths(&staged); + let normalized_actual = canonicalize_session_project_paths(&actual); + let merged = reconcile_session_rows_detailed(&normalized_actual, &expected).map_err( + |conflict| ProjectionStoreError::SessionOutputCollision { + provider: expected.provider.clone(), + session_id: expected.session_id.clone(), + field: conflict.field(), + }, + )?; + if merged != actual { + updates.push(merged); + } + } + drop(overlaps); + write_reconciled_sessions(conn, &updates).await?; + if scanned < REBUILD_PAGE_SIZE { + return Ok(()); + } + cursor = last; } - drop(conflicts); +} + +async fn activate_rebuild_sessions( + conn: &impl Executor, + generation: &str, +) -> ProjectionStoreResult<()> { + reconcile_overlapping_rebuild_sessions(conn, generation).await?; let session_extracts = json_extract_select_list(SESSION_JSON_COLUMN, SESSION_JSON_FIELDS); conn.execute( &format!( @@ -2271,37 +2363,15 @@ async fn activate_rebuild_sessions( transcript_path, metadata_json, parent_session_id, is_subagent, agent_id, parent_tool_use_id ) - SELECT provider, session_id, + SELECT staged.provider, staged.session_id, {session_extracts} - FROM observation_projection_rebuild_sessions - WHERE projector_version = ?1 AND generation = ?2 - ON CONFLICT(provider, session_id) DO UPDATE SET - project_key = CASE - WHEN sessions.project_key = 'user' THEN excluded.project_key - ELSE sessions.project_key END, - project_path = CASE - WHEN sessions.project_path = sessions.project_key THEN excluded.project_path - ELSE sessions.project_path END, - title = COALESCE(sessions.title, excluded.title), - started_at = CASE - WHEN sessions.started_at IS NULL THEN excluded.started_at - WHEN excluded.started_at IS NULL THEN sessions.started_at - ELSE MIN(sessions.started_at, excluded.started_at) END, - ended_at = CASE - WHEN sessions.ended_at IS NULL THEN excluded.ended_at - WHEN excluded.ended_at IS NULL THEN sessions.ended_at - ELSE MAX(sessions.ended_at, excluded.ended_at) END, - transcript_path = COALESCE(sessions.transcript_path, excluded.transcript_path), - metadata_json = CASE - WHEN sessions.metadata_json IS NULL THEN excluded.metadata_json - WHEN excluded.metadata_json IS NULL THEN sessions.metadata_json - ELSE json_patch(excluded.metadata_json, sessions.metadata_json) END, - parent_session_id = COALESCE(sessions.parent_session_id, excluded.parent_session_id), - is_subagent = MAX(sessions.is_subagent, excluded.is_subagent), - agent_id = COALESCE(sessions.agent_id, excluded.agent_id), - parent_tool_use_id = COALESCE( - sessions.parent_tool_use_id, excluded.parent_tool_use_id - )" + FROM observation_projection_rebuild_sessions AS staged + WHERE staged.projector_version = ?1 AND staged.generation = ?2 + AND NOT EXISTS ( + SELECT 1 FROM sessions AS active + WHERE active.provider = staged.provider + AND active.session_id = staged.session_id + )" ), params![SESSION_MESSAGE_PROJECTOR_VERSION, generation], ) @@ -2598,3 +2668,276 @@ async fn activate_rebuild_dispositions( .map(|_| ()) .map_err(|error| storage("activate rebuilt projection dispositions", error)) } + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod activation_tests { + use super::{ + REBUILD_PAGE_SIZE, SESSION_MESSAGE_PROJECTOR_VERSION, activate_rebuild_sessions, + reconcile_overlapping_rebuild_sessions, + }; + use crate::tests::harness::RegisteredGlobalDbHarness; + use tracedecay_runtime_core::db::engine::{Executor, params}; + use tracedecay_store::{ProjectionStoreError, SessionRecord}; + + const SESSION_ID: &str = "002bd803-dc62-46e2-b66a-a61cc282f0dc"; + /// One past the exact-SQL transport's `MAX_QUERY_ROWS` + /// (`tracedecay-rusqlite-runtime/src/exact_sql/mod.rs`), which refuses a + /// result set rather than truncating it. + const OVERLAPS_PAST_TRANSPORT_ROW_CAP: i64 = 10_001; + + fn session(transcript_path: Option<&str>, title: Option<&str>) -> SessionRecord { + SessionRecord { + provider: "cursor".to_owned(), + session_id: SESSION_ID.to_owned(), + project_key: "project.fixture".to_owned(), + project_path: "project.fixture".to_owned(), + title: title.map(str::to_owned), + started_at: Some(1), + ended_at: Some(2), + transcript_path: transcript_path.map(str::to_owned), + metadata_json: None, + parent_session_id: None, + is_subagent: false, + agent_id: None, + parent_tool_use_id: None, + } + } + + async fn stage(transaction: &impl Executor, generation: &str, staged: &SessionRecord) { + transaction + .execute( + "INSERT INTO observation_projection_rebuilds ( + projector_version, generation, frontier_sequence, state + ) VALUES (?1, ?2, 0, 'ready')", + params![SESSION_MESSAGE_PROJECTOR_VERSION, generation], + ) + .await + .unwrap(); + transaction + .execute( + "INSERT INTO observation_projection_rebuild_sessions ( + projector_version, generation, provider, session_id, session_json + ) VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + SESSION_MESSAGE_PROJECTOR_VERSION, + generation, + staged.provider.as_str(), + staged.session_id.as_str(), + serde_json::to_string(staged).unwrap().as_str(), + ], + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn overlap_reconciliation_pages_past_the_transport_row_cap() { + const GENERATION: &str = "generation.session-overlap-paging"; + let harness = RegisteredGlobalDbHarness::open("session-overlap-paging").await; + let mut staged_rows = Vec::with_capacity(OVERLAPS_PAST_TRANSPORT_ROW_CAP as usize); + for index in 0..OVERLAPS_PAST_TRANSPORT_ROW_CAP { + let session_id = format!("session.{index:05}"); + let active = SessionRecord { + session_id: session_id.clone(), + ..session(None, None) + }; + assert!(harness.registered.upsert_session(&active).await); + staged_rows.push(SessionRecord { + session_id, + title: Some(format!("composer {index:05}")), + ..session(None, None) + }); + } + let transaction = harness.registered.begin_write_transaction().await.unwrap(); + transaction + .execute( + "INSERT INTO observation_projection_rebuilds ( + projector_version, generation, frontier_sequence, state + ) VALUES (?1, ?2, 0, 'ready')", + params![SESSION_MESSAGE_PROJECTOR_VERSION, GENERATION], + ) + .await + .unwrap(); + let staged_json = serde_json::to_string(&staged_rows).unwrap(); + transaction + .execute( + "INSERT INTO observation_projection_rebuild_sessions ( + projector_version, generation, provider, session_id, session_json + ) + SELECT ?1, ?2, + json_extract(staged.value, '$.provider'), + json_extract(staged.value, '$.session_id'), + staged.value + FROM json_each(?3) AS staged", + params![ + SESSION_MESSAGE_PROJECTOR_VERSION, + GENERATION, + staged_json.as_str() + ], + ) + .await + .unwrap(); + + reconcile_overlapping_rebuild_sessions(&transaction, GENERATION) + .await + .expect("an overlap larger than one transport page must still reconcile"); + + let mut rows = transaction + .query( + "SELECT COUNT(*) FROM sessions + WHERE provider = 'cursor' AND title LIKE 'composer %'", + (), + ) + .await + .unwrap(); + let reconciled = rows.next().await.unwrap().unwrap().get::(0).unwrap(); + assert_eq!( + reconciled, OVERLAPS_PAST_TRANSPORT_ROW_CAP, + "every overlapping session must be reconciled, not one page of {REBUILD_PAGE_SIZE}", + ); + } + + #[tokio::test] + async fn activation_names_the_session_field_instead_of_a_message_collision() { + let harness = RegisteredGlobalDbHarness::open("session-collision-field").await; + let active = session(Some("/private/old-transcript.jsonl"), None); + assert!(harness.registered.upsert_session(&active).await); + let transaction = harness.registered.begin_write_transaction().await.unwrap(); + let staged = session(Some("/private/new-transcript.jsonl"), None); + stage(&transaction, "generation.session-collision", &staged).await; + + let error = activate_rebuild_sessions(&transaction, "generation.session-collision") + .await + .expect_err("incompatible transcript paths must not activate"); + match &error { + ProjectionStoreError::SessionOutputCollision { + provider, + session_id, + field, + } => { + assert_eq!(provider, "cursor"); + assert_eq!(session_id, SESSION_ID); + assert_eq!(*field, "transcript_path"); + } + other => panic!("session conflict classified as {other}"), + } + let rendered = error.to_string(); + assert!(rendered.contains("transcript_path")); + assert!(!rendered.contains("session:")); + assert!(!rendered.contains("/private/old-transcript.jsonl")); + assert!(!rendered.contains("/private/new-transcript.jsonl")); + + let mut rows = transaction + .query( + "SELECT transcript_path FROM sessions WHERE provider = 'cursor' AND session_id = ?1", + params![SESSION_ID], + ) + .await + .unwrap(); + let persisted = rows + .next() + .await + .unwrap() + .unwrap() + .get::(0) + .unwrap(); + assert_eq!(persisted, "/private/old-transcript.jsonl"); + } + + #[tokio::test] + async fn activation_merges_a_compatible_session_and_inserts_a_new_one() { + let harness = RegisteredGlobalDbHarness::open("session-collision-merge").await; + let active = session(None, None); + assert!(harness.registered.upsert_session(&active).await); + let second_active = SessionRecord { + session_id: "session.second".to_owned(), + ..active.clone() + }; + assert!(harness.registered.upsert_session(&second_active).await); + let transaction = harness.registered.begin_write_transaction().await.unwrap(); + let staged = session(None, Some("Composer session")); + stage(&transaction, "generation.session-merge", &staged).await; + // A second overlap keeps the set-based reconciled write honest: each + // merged row must land on its own session, not the first one twice. + let second_staged = SessionRecord { + title: Some("Second composer session".to_owned()), + ..second_active.clone() + }; + let fresh = SessionRecord { + provider: "cursor".to_owned(), + session_id: "session.fresh".to_owned(), + title: Some("fresh session".to_owned()), + ..active.clone() + }; + for staged in [&second_staged, &fresh] { + transaction + .execute( + "INSERT INTO observation_projection_rebuild_sessions ( + projector_version, generation, provider, session_id, session_json + ) VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + SESSION_MESSAGE_PROJECTOR_VERSION, + "generation.session-merge", + staged.provider.as_str(), + staged.session_id.as_str(), + serde_json::to_string(staged).unwrap().as_str(), + ], + ) + .await + .unwrap(); + } + + activate_rebuild_sessions(&transaction, "generation.session-merge") + .await + .unwrap(); + + let mut rows = transaction + .query( + "SELECT title FROM sessions WHERE provider = 'cursor' AND session_id = ?1", + params![SESSION_ID], + ) + .await + .unwrap(); + let title = rows + .next() + .await + .unwrap() + .unwrap() + .get::(0) + .unwrap(); + assert_eq!(title, "Composer session"); + drop(rows); + let mut rows = transaction + .query( + "SELECT title FROM sessions WHERE provider = 'cursor' AND session_id = 'session.second'", + (), + ) + .await + .unwrap(); + let second_title = rows + .next() + .await + .unwrap() + .unwrap() + .get::(0) + .unwrap(); + assert_eq!(second_title, "Second composer session"); + drop(rows); + let mut rows = transaction + .query( + "SELECT title FROM sessions WHERE provider = 'cursor' AND session_id = 'session.fresh'", + (), + ) + .await + .unwrap(); + let fresh_title = rows + .next() + .await + .unwrap() + .unwrap() + .get::(0) + .unwrap(); + assert_eq!(fresh_title, "fresh session"); + } +} diff --git a/crates/tracedecay-global-db/src/observation_projection/state.rs b/crates/tracedecay-global-db/src/observation_projection/state.rs index c3c911e745..04e38b4b13 100644 --- a/crates/tracedecay-global-db/src/observation_projection/state.rs +++ b/crates/tracedecay-global-db/src/observation_projection/state.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::collections::{BTreeSet, HashMap}; use tracedecay_domain::{CanonicalObservationIdV1, DurableObservationV1}; @@ -687,6 +688,26 @@ async fn message_projection( .ok_or(ProjectionStoreError::ProvenanceCollision) } +/// Session row the output verification compares against. +/// +/// The projection-row batch loads sessions from message rows it found. A +/// missing or relocated message therefore has no batch entry even when the +/// expected session row is durable. That absence is not `row_missing`; the +/// single-output path's [`read_session`] is the authority for it. +pub(in super::super) async fn load_verified_session<'a>( + conn: &impl QueryExecutor, + rows: &'a ProjectionRowsBatch, + provider: &str, + session_id: &str, +) -> ProjectionStoreResult>> { + if let Some(session) = rows.session(provider, session_id) { + return Ok(Some(Cow::Borrowed(session))); + } + Ok(read_session(conn, provider, session_id) + .await? + .map(Cow::Owned)) +} + pub(in super::super) async fn verify_projection_rows( conn: &impl QueryExecutor, projection: &SessionMessageProjection, @@ -789,9 +810,22 @@ pub(in super::super) struct ProjectionOutputAuthority { pub(in super::super) canonical: DurableObservationV1, } +/// The LCM raw twin stored beside one projected message. Not part of the +/// output digest; current provenance still authorizes it because the twin is +/// derived from the same observation. +pub(in super::super) struct ProjectionRawTwin { + pub(in super::super) session_id: String, + pub(in super::super) storage_kind: String, + pub(in super::super) content: String, + pub(in super::super) content_hash: String, + pub(in super::super) snippet_text: String, + pub(in super::super) index_text: String, +} + pub(in super::super) struct ProjectionRowsBatch { sessions: HashMap<(String, String), SessionRecord>, messages: HashMap<(String, String), SessionMessageRecord>, + raw_twins: HashMap<(String, String), ProjectionRawTwin>, } impl ProjectionRowsBatch { @@ -812,6 +846,15 @@ impl ProjectionRowsBatch { self.messages .get(&(provider.to_owned(), message_id.to_owned())) } + + pub(in super::super) fn raw_twin( + &self, + provider: &str, + message_id: &str, + ) -> Option<&ProjectionRawTwin> { + self.raw_twins + .get(&(provider.to_owned(), message_id.to_owned())) + } } pub(in super::super) async fn read_projection_rows_batch( @@ -819,6 +862,7 @@ pub(in super::super) async fn read_projection_rows_batch( outputs: &BTreeSet<(String, String)>, ) -> ProjectionStoreResult { let mut messages = HashMap::with_capacity(outputs.len()); + let mut raw_twins = HashMap::with_capacity(outputs.len()); let requested_keys = outputs.iter().collect::>(); for chunk in requested_keys.chunks(OUTPUT_AUTHORITY_BATCH_KEYS) { let requested = serde_json::to_string( @@ -875,6 +919,55 @@ pub(in super::super) async fn read_projection_rows_batch( message, ); } + drop(rows); + let mut rows = conn + .query( + "SELECT raw.provider, raw.message_id, raw.session_id, raw.storage_kind, + COALESCE(raw.content, ''), raw.content_hash, raw.snippet_text, + raw.index_text + FROM json_each(?1) AS requested + CROSS JOIN lcm_raw_messages AS raw + WHERE raw.provider = json_extract(requested.value, '$.provider') + AND raw.message_id = json_extract(requested.value, '$.message_id')", + params![requested.as_str()], + ) + .await + .map_err(|error| storage("read projected raw twins", error))?; + while let Some(row) = rows + .next() + .await + .map_err(|error| storage("read projected raw twins", error))? + { + let provider = row + .get::(0) + .map_err(|error| storage("decode projected raw twins", error))?; + let message_id = row + .get::(1) + .map_err(|error| storage("decode projected raw twins", error))?; + raw_twins.insert( + (provider, message_id), + ProjectionRawTwin { + session_id: row + .get(2) + .map_err(|error| storage("decode projected raw twins", error))?, + storage_kind: row + .get(3) + .map_err(|error| storage("decode projected raw twins", error))?, + content: row + .get(4) + .map_err(|error| storage("decode projected raw twins", error))?, + content_hash: row + .get(5) + .map_err(|error| storage("decode projected raw twins", error))?, + snippet_text: row + .get(6) + .map_err(|error| storage("decode projected raw twins", error))?, + index_text: row + .get(7) + .map_err(|error| storage("decode projected raw twins", error))?, + }, + ); + } } let session_keys = messages @@ -945,7 +1038,11 @@ pub(in super::super) async fn read_projection_rows_batch( } } - Ok(ProjectionRowsBatch { sessions, messages }) + Ok(ProjectionRowsBatch { + sessions, + messages, + raw_twins, + }) } /// The batched ownership resolution behind [`read_output_authorities`]. @@ -1322,10 +1419,21 @@ pub(super) async fn protected_message_rows_compatible( == Some(expected_hash.as_str()) && payload_ref.is_some_and(|payload_ref| actual.text.contains(payload_ref)); if !external { - let raw = - tracedecay_lcm::schema::load_raw_message(conn, &actual.provider, &actual.message_id) - .await - .map_err(|error| storage("read protected projection output", error))?; + // A twin that fails its own receipt is not a protected rendering of + // this projection. Callers treat that as an ordinary output mismatch + // and, when current provenance uniquely owns the output, rewrite it. + // A database fault is still a fault. + let raw = match tracedecay_lcm::schema::load_raw_message( + conn, + &actual.provider, + &actual.message_id, + ) + .await + { + Ok(raw) => raw, + Err(tracedecay_lcm::LcmError::PayloadIntegrityMismatch) => return Ok(false), + Err(error) => return Err(storage("read protected projection output", error)), + }; let Some(raw) = raw else { return Ok(false); }; @@ -1396,14 +1504,26 @@ pub(super) async fn protected_message_rows_compatible( #[cfg(test)] #[allow(clippy::unwrap_used)] mod reconcile_tests { - #[cfg(unix)] + use std::collections::BTreeSet; + use crate::tests::harness::RegisteredGlobalDbHarness; + use tracedecay_domain::{ + CanonicalObservationEnvelopeV1, ComponentVersion, ObservationId, + ObservationIdentityMaterialV1, ObservationOrderingDomainV1, ObservationScopeV1, + ObservationSourceGenerationV1, ObservationSourceIdentityV1, ObservationSourceRangeV1, + PayloadReferenceV1, RetentionClass, SanitizationReceiptId, SanitizationReceiptRefV1, + SanitizationReceiptV1, SanitizerDispositionV1, SensitivityV1, + }; #[cfg(unix)] use tracedecay_runtime_core::db::engine::params; - use tracedecay_store::SessionRecord; + use tracedecay_store::{ + ObservationProjection, ProjectionStoreError, SessionMessageRecord, SessionRecord, + }; - use super::canonicalize_session_project_paths; - use super::reconcile_session_rows_detailed; + use super::{ + canonicalize_session_project_paths, load_verified_session, read_projection_rows_batch, + reconcile_session_rows_detailed, verify_projection_rows_from_records, + }; fn record(project_path: &str) -> SessionRecord { SessionRecord { @@ -1608,4 +1728,147 @@ mod reconcile_tests { assert_eq!(conflict.field(), "transcript_path"); } + + #[tokio::test] + async fn missing_message_is_an_output_collision_not_a_missing_session() { + let mut fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../../../tests/fixtures/provider_normalization/codex/agent_message.expected_envelope.json" + )) + .unwrap(); + fixture["stable_record_id"] = + serde_json::Value::String("record.missing-message".to_owned()); + fixture["relations"]["session_id"] = + serde_json::Value::String("session.missing-message".to_owned()); + fixture["relations"]["thread_id"] = + serde_json::Value::String("session.missing-message".to_owned()); + fixture["relations"]["message_id"] = + serde_json::Value::String("record.missing-message".to_owned()); + let envelope: CanonicalObservationEnvelopeV1 = serde_json::from_value(fixture).unwrap(); + let source = ObservationSourceIdentityV1::for_provider( + envelope.provider().clone(), + envelope.relations().session_id().clone(), + ) + .unwrap(); + let payload = serde_json::to_value(&envelope).unwrap(); + let receipt = SanitizationReceiptV1::new( + SanitizationReceiptRefV1::new( + SanitizationReceiptId::new("receipt.missing-message").unwrap(), + ComponentVersion::new("sanitizer.missing-message.v1").unwrap(), + ) + .unwrap(), + SanitizerDispositionV1::Accepted, + SensitivityV1::NonSensitive, + Some(PayloadReferenceV1::for_payload(&payload).unwrap()), + ) + .unwrap(); + let observation = tracedecay_domain::DurableObservationV1::new( + ObservationIdentityMaterialV1::for_native_record( + source, + ObservationScopeV1::Profile, + ObservationSourceGenerationV1::new(1).unwrap(), + ObservationSourceRangeV1::new(0, 100).unwrap(), + ObservationOrderingDomainV1::FileBytes, + ObservationId::new("record.missing-message").unwrap(), + ) + .unwrap(), + receipt, + RetentionClass::new("retention.missing-message").unwrap(), + payload, + ) + .unwrap(); + let session = SessionRecord { + provider: "codex".to_owned(), + session_id: "session.missing-message".to_owned(), + project_key: "user".to_owned(), + project_path: "user".to_owned(), + title: None, + started_at: Some(1), + ended_at: Some(2), + transcript_path: None, + metadata_json: None, + parent_session_id: None, + is_subagent: false, + agent_id: None, + parent_tool_use_id: None, + }; + let message = SessionMessageRecord { + provider: "codex".to_owned(), + message_id: "record.missing-message".to_owned(), + session_id: "session.missing-message".to_owned(), + role: "assistant".to_owned(), + timestamp: Some(1), + ordinal: 0, + text: "The billing pipeline regression is fixed.".to_owned(), + kind: None, + model: None, + tool_names: None, + source_path: None, + source_offset: None, + metadata_json: None, + }; + let projection = ObservationProjection::for_message(&observation, session, message) + .unwrap() + .message() + .expect("explicit message projection") + .clone(); + let message = projection.message(); + let session = projection.session(); + assert_eq!(message.provider, "codex"); + assert_eq!(message.message_id, "record.missing-message"); + assert_eq!(session.session_id, "session.missing-message"); + let outputs = BTreeSet::from([(message.provider.clone(), message.message_id.clone())]); + + let harness = RegisteredGlobalDbHarness::open("missing-message-collision").await; + let absent = harness.registered.read_snapshot().await.unwrap(); + let batch = read_projection_rows_batch(&absent, &outputs).await.unwrap(); + assert!(batch.message("codex", "record.missing-message").is_none()); + assert!( + load_verified_session(&absent, &batch, "codex", "session.missing-message") + .await + .unwrap() + .is_none() + ); + let missing_session = verify_projection_rows_from_records(&absent, &projection, None, None) + .await + .expect_err("a projection with no stored session is a session collision"); + assert!(matches!( + missing_session, + ProjectionStoreError::SessionOutputCollision { + field: "row_missing", + .. + } + )); + + assert!(harness.registered.upsert_session(session).await); + let present = harness.registered.read_snapshot().await.unwrap(); + let batch = read_projection_rows_batch(&present, &outputs) + .await + .unwrap(); + assert!( + batch.session("codex", "session.missing-message").is_none(), + "the message-keyed batch still does not see a session the message row never named" + ); + let loaded = load_verified_session(&present, &batch, "codex", "session.missing-message") + .await + .unwrap() + .expect("the durable session row is not missing"); + let missing_message = verify_projection_rows_from_records( + &present, + &projection, + Some(loaded.as_ref()), + batch.message("codex", "record.missing-message"), + ) + .await + .expect_err("a missing message with a live session is an output collision"); + match missing_message { + ProjectionStoreError::OutputCollision { + provider, + message_id, + } => { + assert_eq!(provider, "codex"); + assert_eq!(message_id, "record.missing-message"); + } + other => panic!("missing message classified as {other}"), + } + } } diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs index 565c0a309d..f247622745 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs @@ -2,9 +2,11 @@ use std::collections::{BTreeSet, HashMap}; use futures_util::future::try_join_all; use tracedecay_domain::DurableObservationV1; +use tracedecay_privacy::sanitize_lcm_payload_text; use tracedecay_store::{ ObservationProjection, ProjectionSkipReason, ProjectionStoreError, SESSION_MESSAGE_PROJECTOR_VERSION, SessionMessageProjection, WorkflowFactProjection, + stored_message_is_shipped_release_rendering, }; use crate::observation_projection::{ProjectionOutputAuthority, ProjectionRowsBatch}; @@ -816,32 +818,49 @@ async fn validate_message_projection_row( )? == StoredProvenanceRendering::Current { // Convergence supersedes an existing output row; it never inserts one. - // Both repair arms below therefore require the row to be there: a - // vanished output stays the hard failure #1775 and #1781 both promised, - // instead of a recorded repair that writes nothing. The batch also - // derives its session keys from the message rows it found, so a missing - // message is reported as a missing *session* row, which is why this - // guard has to cover the session arm too. + // A vanished message stays a hard failure. Session repair is only the + // uniquely owned current output whose session row is absent. let owner_message = owner_projection.message(); let output_row_present = resolved .projection_rows .message(&owner_message.provider, &owner_message.message_id) .is_some(); match verify_owner_output_rows(conn, resolved, &owner_projection).await { - Ok(()) => {} + Ok(()) => { + // Message equality is not the whole output. The raw twin is + // derived from the same observation and is not covered by the + // digest, so a matching message can still sit on a stale twin. + // Protected rows are not this arm: their stored message differs + // from the projection, and that compatibility already checked + // the twin. + if resolved + .projection_rows + .message(&owner_message.provider, &owner_message.message_id) + .is_some_and(|stored| stored == owner_message) + && owned_raw_twin_needs_rewrite(&owner_projection, &resolved.projection_rows)? + { + resolved.released.record(&owner_projection); + } + } Err(ProjectionStoreError::OutputCollision { provider, message_id, }) if output_row_present && provider == owner_projection.message().provider - && message_id == owner_projection.message().message_id => + && message_id == owner_projection.message().message_id + && resolved + .projection_rows + .message(&provider, &message_id) + .is_some_and(|stored| { + stored_message_is_shipped_release_rendering(&authority.canonical, stored) + }) => { - // Ownership was validated above, the immutable observation - // re-derived this projection, and its provenance already - // carries the projection's current digest. The mutable output - // row is the only stale member, an interrupted/older write - // shape observed in ProfileSessions. Finish that write in the - // same convergence ledger used for released renderings. + // Provenance already carries this binary's digest, and the + // mutable row is still the rendering a shipped release wrote + // for this observation: the write that stamped the digest did + // not finish. Finish it on the released-rendering ledger. A + // body that matches neither rendering is tamper and falls + // through to the hard failure below. resolved.released.record(&owner_projection); } Err(ProjectionStoreError::SessionOutputCollision { @@ -877,12 +896,17 @@ async fn verify_owner_output_rows( ) -> std::result::Result<(), ProjectionStoreError> { let session = owner.session(); let message = owner.message(); + let session_row = crate::observation_projection::load_verified_session( + conn, + &resolved.projection_rows, + &session.provider, + &session.session_id, + ) + .await?; crate::observation_projection::verify_projection_rows_from_records( conn, owner, - resolved - .projection_rows - .session(&session.provider, &session.session_id), + session_row.as_deref(), resolved .projection_rows .message(&message.provider, &message.message_id), @@ -890,6 +914,45 @@ async fn verify_owner_output_rows( .await } +/// Whether the LCM raw twin of a message that already matches this projection +/// is not the twin a fresh projection write would store. +/// +/// Hermes projections have no raw twin. A sanitizer quarantine is itself the +/// current rendering, so the caller records the projection for the same +/// converge path a fresh capture uses. A sanitizer fault stays a typed refusal. +fn owned_raw_twin_needs_rewrite( + projection: &SessionMessageProjection, + rows: &ProjectionRowsBatch, +) -> tracedecay_domain::errors::Result { + let message = projection.message(); + if message.provider == "hermes" { + return Ok(false); + } + let expected = match sanitize_lcm_payload_text(&message.text) { + Ok(sanitized) => sanitized.sanitized_text().to_owned(), + Err(error) if error.is_quarantine_verdict() => return Ok(true), + Err(error) => { + return Err(authority_violation(format!( + "projection raw twin sanitizer failed: {error}" + ))); + } + }; + let Some(raw) = rows.raw_twin(&message.provider, &message.message_id) else { + return Ok(true); + }; + // The derived columns are pure functions of the same sanitized body, so a + // twin whose content matches can still carry a hash that fails hydration + // with `PayloadIntegrityMismatch` or retrieval text the projector never + // wrote. Compare what a fresh write stores, not content alone. + Ok(raw.storage_kind != "inline" + || raw.session_id != message.session_id + || raw.content != expected + || raw.content_hash != tracedecay_lcm::retrieval_content::projected_content_hash(&expected) + || raw.snippet_text + != tracedecay_lcm::retrieval_content::derived_text_for_snippet(&expected) + || raw.index_text != tracedecay_lcm::retrieval_content::derived_text_for_index(&expected)) +} + #[allow(clippy::too_many_arguments)] async fn validate_message_projection( conn: &impl QueryExecutor, diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs index 760c1dcbd9..7b6e6d096c 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/released_rendering.rs @@ -33,7 +33,10 @@ //! that owns the transaction. A row disagreeing on identity, anchor, receipt, //! output provider or message id, or carrying a digest that matches neither //! this binary's output nor its own output row is not a rendering difference, -//! and stays refused, named. +//! and stays refused, named. A current digest over a mutable row that is still +//! that shipped rendering is the same admission: the write that stamped the +//! digest did not finish. A row that matches neither rendering is tamper and +//! stays refused. //! //! Convergence has two outcomes because rendering does. Some released //! renderings are content the current LCM privacy sanitizer withholds, a @@ -662,6 +665,172 @@ mod tests { assert_eq!(stored_output(&snapshot, RECORD_ID).await, current); } + /// The digest covers the message, not its LCM raw twin. A twin can be + /// rewritten under a still-current message and provenance; reopen has to + /// restore the twin a fresh projection write stores. + #[tokio::test] + async fn current_provenance_repairs_a_stale_raw_twin() { + let directory = TempDir::new().unwrap(); + let runtime = HostAdmissionTestRuntimeV1::profile(directory.path()) + .await + .unwrap(); + seed(&runtime, &observation()).await.unwrap(); + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .expect("registered profile database"); + let snapshot = database.read_snapshot().await.unwrap(); + let current = stored_output(&snapshot, RECORD_ID).await; + drop(snapshot); + + let transaction = database + .runtime_database() + .begin_write_transaction("stale the raw twin under current provenance") + .await + .unwrap(); + let updated = transaction + .execute( + "UPDATE lcm_raw_messages + SET content = 'stale raw body', content_hash = 'stale', + snippet_text = 'stale raw body', index_text = 'stale raw body' + WHERE provider = 'codex' AND message_id = ?1", + tracedecay_runtime_core::params![RECORD_ID], + ) + .await + .expect("stale the raw twin"); + assert_eq!(updated, 1); + transaction.commit().await.unwrap(); + + let snapshot = database.read_snapshot().await.unwrap(); + let stale = stored_output(&snapshot, RECORD_ID).await; + drop(snapshot); + assert_eq!(stale.digest, current.digest); + assert_eq!(stale.text, current.text); + assert_eq!(stale.raw_index_text, "stale raw body"); + + super::super::ensure_authority_invariants(database.runtime_database(), false, false) + .await + .expect("current provenance must repair its stale raw twin"); + + let snapshot = database.read_snapshot().await.unwrap(); + assert_eq!(stored_output(&snapshot, RECORD_ID).await, current); + } + + /// A twin whose content survived but whose derived columns did not still + /// fails hydration with `PayloadIntegrityMismatch`. Content equality alone + /// is not the twin a fresh projection write stores. + #[tokio::test] + async fn current_provenance_repairs_a_raw_twin_with_a_stale_hash() { + let directory = TempDir::new().unwrap(); + let runtime = HostAdmissionTestRuntimeV1::profile(directory.path()) + .await + .unwrap(); + seed(&runtime, &observation()).await.unwrap(); + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .expect("registered profile database"); + let snapshot = database.read_snapshot().await.unwrap(); + let current = stored_output(&snapshot, RECORD_ID).await; + drop(snapshot); + + let transaction = database + .runtime_database() + .begin_write_transaction("stale the raw twin derivations") + .await + .unwrap(); + let updated = transaction + .execute( + "UPDATE lcm_raw_messages + SET content_hash = 'stale', index_text = 'stale index' + WHERE provider = 'codex' AND message_id = ?1", + tracedecay_runtime_core::params![RECORD_ID], + ) + .await + .expect("stale the derived columns"); + assert_eq!(updated, 1); + transaction.commit().await.unwrap(); + + super::super::ensure_authority_invariants(database.runtime_database(), false, false) + .await + .expect("current provenance must repair a twin with stale derivations"); + + let snapshot = database.read_snapshot().await.unwrap(); + assert_eq!(stored_output(&snapshot, RECORD_ID).await, current); + let mut rows = snapshot + .query( + "SELECT content_hash, content FROM lcm_raw_messages + WHERE provider = 'codex' AND message_id = ?1", + tracedecay_runtime_core::params![RECORD_ID], + ) + .await + .expect("read the repaired twin"); + let row = rows + .next() + .await + .expect("read the repaired twin") + .expect("raw twin row"); + assert_eq!( + row.get::(0).unwrap(), + tracedecay_lcm::retrieval_content::projected_content_hash( + &row.get::(1).unwrap() + ), + "the repaired twin must carry the hash its content hydrates against" + ); + } + + /// The repair above is the shipped rendering, not any disagreement under + /// current provenance. A body neither this binary nor a release wrote is + /// tamper: the audit refuses it and does not rewrite the row. + #[tokio::test] + async fn current_provenance_refuses_a_tampered_output_row() { + let directory = TempDir::new().unwrap(); + let runtime = HostAdmissionTestRuntimeV1::profile(directory.path()) + .await + .unwrap(); + seed(&runtime, &observation()).await.unwrap(); + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .expect("registered profile database"); + + let transaction = database + .runtime_database() + .begin_write_transaction("tamper the projected message body") + .await + .unwrap(); + let updated = transaction + .execute( + "UPDATE session_messages SET text = 'tampered projection body' + WHERE provider = 'codex' AND message_id = ?1", + tracedecay_runtime_core::params![RECORD_ID], + ) + .await + .expect("tamper projected message"); + assert_eq!(updated, 1); + transaction + .execute("DELETE FROM authority_audit_checkpoints", ()) + .await + .unwrap(); + transaction.commit().await.unwrap(); + + let error = + super::super::ensure_authority_invariants(database.runtime_database(), true, false) + .await + .expect_err( + "a tampered projected message under current provenance must stay refused", + ); + let message = error.to_string(); + assert!( + message.contains("projection output rows disagree with deterministic output"), + "{message}" + ); + + let snapshot = database.read_snapshot().await.unwrap(); + assert_eq!( + stored_output(&snapshot, RECORD_ID).await.text, + "tampered projection body", + "refusal must not rewrite the tampered row" + ); + } + #[tokio::test] async fn current_provenance_restores_its_missing_session_row() { let directory = TempDir::new().unwrap(); diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/triggers.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/triggers.rs index 7defbbff58..0924d3a151 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/triggers.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/triggers.rs @@ -227,6 +227,38 @@ const PROJECTION_AUDIT_INVALIDATION: &[Trigger] = &[ WHERE audit_name = 'observation-authority'; END", }, + // The message-row triggers above do not see the LCM raw twin. A twin can + // drift (content, session identity) while the message row and the current + // provenance digest stay put, and the trusted checkpoint would then skip + // it forever. Invalidate on the same ownership predicate. + Trigger { + name: "projection_raw_audit_invalidate_update_v1", + table: "lcm_raw_messages", + create_sql: "CREATE TRIGGER projection_raw_audit_invalidate_update_v1 + AFTER UPDATE ON lcm_raw_messages + WHEN EXISTS ( + SELECT 1 FROM observation_projection_provenance + WHERE output_provider = OLD.provider + AND output_message_id = OLD.message_id + ) BEGIN + DELETE FROM authority_audit_checkpoints + WHERE audit_name = 'observation-authority'; + END", + }, + Trigger { + name: "projection_raw_audit_invalidate_delete_v1", + table: "lcm_raw_messages", + create_sql: "CREATE TRIGGER projection_raw_audit_invalidate_delete_v1 + AFTER DELETE ON lcm_raw_messages + WHEN EXISTS ( + SELECT 1 FROM observation_projection_provenance + WHERE output_provider = OLD.provider + AND output_message_id = OLD.message_id + ) BEGIN + DELETE FROM authority_audit_checkpoints + WHERE audit_name = 'observation-authority'; + END", + }, Trigger { name: "projection_checkpoint_audit_invalidate_regression_v1", table: "observation_projection_checkpoints", @@ -1650,6 +1682,15 @@ struct ReleasedV3TriggerDrift { released: &'static str, } +/// Triggers added after the v3 inventory. A released store is admitted on the +/// published bodies, then schema convergence installs these and the missing +/// contract forces the exhaustive repair pass. Requiring them at admission +/// would reset every beta.25–beta.37 profile. +const POST_RELEASED_V3_TRIGGERS: &[&str] = &[ + "projection_raw_audit_invalidate_update_v1", + "projection_raw_audit_invalidate_delete_v1", +]; + const RELEASED_V3_TRIGGER_DRIFT: &[ReleasedV3TriggerDrift] = &[ ReleasedV3TriggerDrift { trigger: "session_refresh_progress_insert_guard_v1", @@ -1721,6 +1762,9 @@ pub async fn released_v3_invariant_triggers_intact( let released = released_v3_trigger_contracts()?; for invariant in INVARIANTS { for trigger in invariant.triggers { + if POST_RELEASED_V3_TRIGGERS.contains(&trigger.name) { + continue; + } let expected = released .iter() .find(|(name, _)| *name == trigger.name) diff --git a/crates/tracedecay-global-db/src/schema_stages.rs b/crates/tracedecay-global-db/src/schema_stages.rs index 34cc15ba64..56aeff0f89 100644 --- a/crates/tracedecay-global-db/src/schema_stages.rs +++ b/crates/tracedecay-global-db/src/schema_stages.rs @@ -848,6 +848,22 @@ async fn install_registered_schema_stage_sequence( .execute_batch(RUNTIME_LEDGER_SCHEMA) .await .map_err(|error| global_db_operation_error("initialize runtime writer ledger", error))?; + // Projection raw-twin triggers sit on `lcm_raw_messages`. The table has to + // exist before those triggers are created, including on a fresh store + // whose authority triggers are installed in this same transaction. + tracedecay_lcm::schema::ensure_lcm_schema_in_transaction(transaction) + .await + .map_err(|error| match error { + tracedecay_lcm::LcmError::ProfileResetRequired { + found_version, + required_version, + } => tracedecay_domain::errors::TraceDecayError::ProfileResetRequired { + component: "LCM", + found_version, + required_version, + }, + error => global_db_operation_error("initialize LCM schema", error), + })?; // `force_exhaustive` means admission observed damaged or missing guard // triggers (for example a dropped guarded table takes its triggers with // it). Reinstall them here so the post-commit contract validation sees a @@ -865,20 +881,6 @@ async fn install_registered_schema_stage_sequence( )); } } - - tracedecay_lcm::schema::ensure_lcm_schema_in_transaction(transaction) - .await - .map_err(|error| match error { - tracedecay_lcm::LcmError::ProfileResetRequired { - found_version, - required_version, - } => tracedecay_domain::errors::TraceDecayError::ProfileResetRequired { - component: "LCM", - found_version, - required_version, - }, - error => global_db_operation_error("initialize LCM schema", error), - })?; tracedecay_sessions::runtime::git_correlation::ensure_git_correlation_receipt_schema_in_transaction( transaction, ) diff --git a/crates/tracedecay-host-admission/src/replay.rs b/crates/tracedecay-host-admission/src/replay.rs index ceffe51442..b9b1588c39 100644 --- a/crates/tracedecay-host-admission/src/replay.rs +++ b/crates/tracedecay-host-admission/src/replay.rs @@ -7,7 +7,7 @@ use std::time::Duration; -use tracedecay_sessions::admission::HostAdmissionOutcome; +use tracedecay_sessions::admission::{HostAdmissionOutcome, HostAdmissionStatus}; const MAX_BACKOFF: Duration = Duration::from_secs(2); const INITIAL_BACKOFF: Duration = Duration::from_millis(25); @@ -28,6 +28,7 @@ pub fn replay_backoff(attempt: u32, shift_cap: u32) -> Duration { } /// How a worker should proceed after one replay pass. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ReplayPassDecision { /// The spool shrank and more work remains. Yield and re-run immediately. ProgressPending, @@ -37,15 +38,33 @@ pub enum ReplayPassDecision { Stop, /// Re-evaluate the work condition without backoff. Requeue, + /// Closed `NotApplicable` left the spool unchanged. Stop until the next + /// kick without backoff and without a failure log. + TerminalNoop, } /// Classify one replay pass from its pending-count delta and outcome. +/// +/// `NotApplicable` already closes the replay record. It must not enter the +/// retryable backoff arm, even when `is_replay_progress` is true and the +/// spool did not shrink: that arm is for work that may succeed later. A shrink +/// with records still pending continues immediately; a drained spool requeues; +/// an unchanged spool stops until the next external kick. pub fn classify_replay_pass( pending_before: usize, pending_after: usize, outcome: &HostAdmissionOutcome, ) -> ReplayPassDecision { let made_progress = pending_after < pending_before; + if outcome.status == HostAdmissionStatus::NotApplicable { + if made_progress && pending_after > 0 { + return ReplayPassDecision::ProgressPending; + } + if pending_after == 0 { + return ReplayPassDecision::Requeue; + } + return ReplayPassDecision::TerminalNoop; + } if made_progress && pending_after > 0 { ReplayPassDecision::ProgressPending } else if !made_progress @@ -58,3 +77,42 @@ pub fn classify_replay_pass( ReplayPassDecision::Requeue } } + +#[cfg(test)] +mod tests { + use super::{ReplayPassDecision, classify_replay_pass}; + use tracedecay_sessions::admission::HostAdmissionOutcome; + + #[test] + fn not_applicable_is_a_terminal_noop_unless_the_spool_actually_moves() { + let closed = HostAdmissionOutcome::not_applicable("code_index_not_applicable"); + let mut flagged_retryable = closed.clone(); + flagged_retryable.retryable = true; + + assert_eq!( + classify_replay_pass(2, 2, &closed), + ReplayPassDecision::TerminalNoop + ); + assert_eq!( + classify_replay_pass(2, 3, &flagged_retryable), + ReplayPassDecision::TerminalNoop + ); + assert_eq!( + classify_replay_pass(2, 1, &closed), + ReplayPassDecision::ProgressPending + ); + assert_eq!( + classify_replay_pass(1, 0, &closed), + ReplayPassDecision::Requeue + ); + + assert_eq!( + classify_replay_pass(2, 2, &HostAdmissionOutcome::accepted_for_replay()), + ReplayPassDecision::Backoff + ); + assert_eq!( + classify_replay_pass(2, 2, &HostAdmissionOutcome::spool_corrupted()), + ReplayPassDecision::Stop + ); + } +} diff --git a/crates/tracedecay-hotpath-guard/Cargo.toml b/crates/tracedecay-hotpath-guard/Cargo.toml new file mode 100644 index 0000000000..72f5df1a5b --- /dev/null +++ b/crates/tracedecay-hotpath-guard/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "tracedecay-hotpath-guard" +version = "0.1.0" +publish = false +edition.workspace = true +license = "MIT" +description = "Process-boundary Hotpath display limit for the shipped binary" +repository = "https://github.com/ScriptedAlchemy/tracedecay" + +[lib] +doctest = false + +[features] +# Same activation as the shipped binary: the runtime crate stays crates.io +# hotpath. `hotpath-mcp` is only so the live-query proof can start the server. +hotpath = ["hotpath/hotpath"] +hotpath-mcp = ["hotpath", "hotpath/hotpath-mcp"] + +[dependencies] +hotpath.workspace = true + +[dev-dependencies] +hotpath.workspace = true +serde_json = "1" + +[[test]] +name = "functions_limit_live" +path = "tests/functions_limit_live.rs" +required-features = ["hotpath", "hotpath-mcp"] diff --git a/crates/tracedecay-hotpath-guard/src/lib.rs b/crates/tracedecay-hotpath-guard/src/lib.rs new file mode 100644 index 0000000000..89121cb917 --- /dev/null +++ b/crates/tracedecay-hotpath-guard/src/lib.rs @@ -0,0 +1,76 @@ +//! Process-boundary display cap for the shipped Hotpath guard. +//! +//! Published hotpath 0.24 reads `HOTPATH_FUNCTIONS_LIMIT` (then `HOTPATH_LIMIT`) +//! only when the exit report is built. Live `functions_timing` and +//! `functions_alloc` use the builder limit snapshotted when the guard starts. +//! The shipped binary applies this before that snapshot so a limit already in +//! the process environment is what MCP returns. `0` stays unlimited. A value +//! set after the process is running cannot resize the already started worker. + +/// Applies the functions display limit from the process environment, if set. +/// +/// Unset or unparsable variables leave the builder unchanged, matching the +/// exit report's fallback to the builder default. +pub fn with_functions_display_limit( + builder: hotpath::HotpathGuardBuilder, +) -> hotpath::HotpathGuardBuilder { + match functions_display_limit() { + Some(limit) => builder.functions_limit(limit), + None => builder, + } +} + +fn functions_display_limit() -> Option { + parse_usize_env("HOTPATH_FUNCTIONS_LIMIT").or_else(|| parse_usize_env("HOTPATH_LIMIT")) +} + +fn parse_usize_env(name: &str) -> Option { + std::env::var(name).ok().and_then(|raw| raw.parse().ok()) +} + +#[cfg(test)] +mod tests { + use super::functions_display_limit; + + /// One test owns both variables: they are process-global, and the live MCP + /// proof (`tests/functions_limit_live.rs`) only runs under `hotpath`. + #[test] + fn functions_limit_precedes_the_global_limit_and_ignores_junk() { + let set = |name: &str, value: Option<&str>| unsafe { + match value { + Some(value) => std::env::set_var(name, value), + None => std::env::remove_var(name), + } + }; + + set("HOTPATH_FUNCTIONS_LIMIT", None); + set("HOTPATH_LIMIT", None); + assert_eq!(functions_display_limit(), None, "unset leaves the builder"); + + set("HOTPATH_LIMIT", Some("7")); + assert_eq!(functions_display_limit(), Some(7), "HOTPATH_LIMIT is used"); + + set("HOTPATH_FUNCTIONS_LIMIT", Some("2")); + assert_eq!( + functions_display_limit(), + Some(2), + "HOTPATH_FUNCTIONS_LIMIT wins" + ); + + set("HOTPATH_FUNCTIONS_LIMIT", Some("0")); + assert_eq!(functions_display_limit(), Some(0), "0 stays unlimited"); + + set("HOTPATH_FUNCTIONS_LIMIT", Some("not-a-number")); + assert_eq!( + functions_display_limit(), + Some(7), + "junk falls back to HOTPATH_LIMIT" + ); + + set("HOTPATH_LIMIT", Some("-1")); + assert_eq!(functions_display_limit(), None, "junk in both leaves it"); + + set("HOTPATH_FUNCTIONS_LIMIT", None); + set("HOTPATH_LIMIT", None); + } +} diff --git a/crates/tracedecay-hotpath-guard/tests/functions_limit_live.rs b/crates/tracedecay-hotpath-guard/tests/functions_limit_live.rs new file mode 100644 index 0000000000..b626db2b31 --- /dev/null +++ b/crates/tracedecay-hotpath-guard/tests/functions_limit_live.rs @@ -0,0 +1,344 @@ +//! Live MCP `functions_timing` must honor `HOTPATH_FUNCTIONS_LIMIT` when the +//! shipped guard applies it before the profiler starts. +//! +//! The builder limit is unlimited. Without `with_functions_display_limit`, +//! hotpath 0.24's worker keeps that unlimited snapshot and the tool returns +//! every measured function. The env is set before the guard starts, which is +//! the process boundary the shipped binary has. + +use std::io::{Read, Write}; +use std::net::{SocketAddr, TcpStream}; +use std::time::Duration; + +use tracedecay_hotpath_guard::with_functions_display_limit; + +#[hotpath::measure] +fn slow_long() { + std::thread::sleep(Duration::from_millis(200)); +} + +#[hotpath::measure] +fn slow_mid() { + std::thread::sleep(Duration::from_millis(20)); +} + +#[hotpath::measure] +fn slow_short() { + std::thread::sleep(Duration::from_millis(5)); +} + +#[hotpath::measure] +fn slow_tiny() { + std::thread::sleep(Duration::from_millis(1)); +} + +#[test] +fn live_mcp_functions_timing_honors_functions_limit() { + let port = free_port(); + let report_path = std::env::temp_dir().join(format!( + "hotpath-functions-limit-{}.json", + std::process::id() + )); + let _ = std::fs::remove_file(&report_path); + + unsafe { + std::env::set_var("HOTPATH_EXCLUDE_WRAPPER", "1"); + std::env::set_var("HOTPATH_METRICS_SERVER_OFF", "1"); + std::env::set_var("HOTPATH_MCP_PORT", port.to_string()); + std::env::set_var("HOTPATH_FUNCTIONS_LIMIT", "2"); + std::env::set_var("HOTPATH_OUTPUT_FORMAT", "json"); + std::env::set_var("HOTPATH_OUTPUT_PATH", &report_path); + std::env::remove_var("HOTPATH_LIMIT"); + std::env::remove_var("HOTPATH_REPORT"); + std::env::remove_var("HOTPATH_MCP_AUTH_TOKEN"); + } + + let guard = with_functions_display_limit( + hotpath::HotpathGuardBuilder::new("functions-limit-live") + .functions_limit(0) + .format(hotpath::Format::Json) + .output_path(&report_path), + ) + .build(); + + slow_long(); + slow_mid(); + slow_short(); + slow_tiny(); + + let session = initialize(&port); + let names = wait_for_functions(&port, &session, |got| !got.is_empty()); + assert_live_limit(&names); + + drop(guard); + + let report = std::fs::read_to_string(&report_path).unwrap_or_else(|error| { + panic!("exit report missing at {}: {error}", report_path.display()) + }); + let report: serde_json::Value = serde_json::from_str(&report) + .unwrap_or_else(|error| panic!("exit report is not JSON: {error}\n{report}")); + let exit_names = names_from_list( + report + .get("functions_timing") + .unwrap_or_else(|| panic!("exit report has no functions_timing: {report}")), + ); + assert_live_limit(&exit_names); + let _ = std::fs::remove_file(&report_path); +} + +fn assert_live_limit(names: &[String]) { + assert_eq!( + names.len(), + 2, + "HOTPATH_FUNCTIONS_LIMIT=2 must keep the two slowest functions, got {names:?}" + ); + assert!( + names.iter().any(|name| name.contains("slow_long")), + "missing slow_long in {names:?}" + ); + assert!( + names.iter().any(|name| name.contains("slow_mid")), + "missing slow_mid in {names:?}" + ); + assert!( + names + .iter() + .all(|name| !name.contains("slow_short") && !name.contains("slow_tiny")), + "faster functions leaked past the limit: {names:?}" + ); +} + +fn free_port() -> u16 { + std::net::TcpListener::bind("127.0.0.1:0") + .expect("bind ephemeral port") + .local_addr() + .expect("ephemeral local addr") + .port() +} + +fn initialize(port: &u16) -> String { + let body = r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"functions-limit-live","version":"0"}}}"#; + let mut last = String::new(); + for _ in 0..50 { + match post(port, None, body) { + Ok(response) if response.status == 200 => { + let session = response.header("mcp-session-id").unwrap_or_else(|| { + panic!( + "initialize response missing mcp-session-id: status {} body {}", + response.status, response.body + ) + }); + let notified = post( + port, + Some(session.as_str()), + r#"{"jsonrpc":"2.0","method":"notifications/initialized"}"#, + ) + .unwrap_or_else(|error| panic!("initialized notification failed: {error}")); + assert!( + notified.status == 202 || notified.status == 200, + "initialized notification status {}: {}", + notified.status, + notified.body + ); + return session; + } + Ok(response) => { + last = format!("status {} body {}", response.status, response.body); + } + Err(error) => last = error, + } + std::thread::sleep(Duration::from_millis(20)); + } + panic!("MCP server did not accept initialize on port {port}: {last}"); +} + +fn wait_for_functions(port: &u16, session: &str, ready: impl Fn(&[String]) -> bool) -> Vec { + let mut last = String::new(); + for _ in 0..40 { + match post( + port, + Some(session), + r#"{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"functions_timing","arguments":{}}}"#, + ) { + Ok(response) if response.status == 200 => { + let payload = rpc_result(&response.body); + if payload + .pointer("/result/isError") + .and_then(|value| value.as_bool()) + == Some(true) + { + panic!("functions_timing returned an error: {}", response.body); + } + let text = payload + .pointer("/result/content/0/text") + .and_then(|value| value.as_str()) + .unwrap_or_else(|| { + panic!( + "functions_timing response has no text content: {}", + response.body + ) + }); + let list: serde_json::Value = serde_json::from_str(text).unwrap_or_else(|error| { + panic!("functions_timing text is not JSON: {error}\n{text}") + }); + let names = names_from_list(&list); + if ready(&names) { + return names; + } + last = format!("functions_timing not ready: {names:?}"); + } + Ok(response) => { + last = format!("status {} body {}", response.status, response.body); + } + Err(error) => last = error, + } + std::thread::sleep(Duration::from_millis(25)); + } + panic!("functions_timing did not return the expected measurements: {last}"); +} + +fn names_from_list(list: &serde_json::Value) -> Vec { + list.get("data") + .and_then(|data| data.as_array()) + .unwrap_or_else(|| panic!("function list has no data array: {list}")) + .iter() + .map(|entry| { + entry + .get("name") + .and_then(|name| name.as_str()) + .unwrap_or_else(|| panic!("function entry has no name: {entry}")) + .to_string() + }) + .collect() +} + +fn rpc_result(body: &str) -> serde_json::Value { + let trimmed = body.trim(); + if trimmed.starts_with('{') { + return serde_json::from_str(trimmed) + .unwrap_or_else(|error| panic!("MCP body is not JSON: {error}\n{body}")); + } + let mut found = None; + for line in trimmed.lines() { + let Some(data) = line.trim().strip_prefix("data:") else { + continue; + }; + let data = data.trim(); + if !data.starts_with('{') { + continue; + } + let value: serde_json::Value = serde_json::from_str(data) + .unwrap_or_else(|error| panic!("MCP event is not JSON: {error}\n{data}")); + if value.get("result").is_some() || value.get("error").is_some() { + found = Some(value); + } + } + found.unwrap_or_else(|| panic!("MCP response had no JSON-RPC payload:\n{body}")) +} + +struct HttpResponse { + status: u16, + headers: Vec<(String, String)>, + body: String, +} + +impl HttpResponse { + fn header(&self, name: &str) -> Option { + self.headers.iter().find_map(|(key, value)| { + key.eq_ignore_ascii_case(name) + .then(|| value.trim().to_string()) + }) + } +} + +fn post(port: &u16, session: Option<&str>, body: &str) -> Result { + let address = SocketAddr::from(([127, 0, 0, 1], *port)); + let mut stream = TcpStream::connect_timeout(&address, Duration::from_millis(200)) + .map_err(|error| error.to_string())?; + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .map_err(|error| error.to_string())?; + stream + .set_write_timeout(Some(Duration::from_secs(2))) + .map_err(|error| error.to_string())?; + + let mut request = format!( + "POST /mcp HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nContent-Type: application/json\r\nAccept: application/json, text/event-stream\r\nMCP-Protocol-Version: 2024-11-05\r\nContent-Length: {}\r\nConnection: close\r\n", + body.len() + ); + if let Some(session) = session { + request.push_str(&format!("mcp-session-id: {session}\r\n")); + } + request.push_str("\r\n"); + request.push_str(body); + stream + .write_all(request.as_bytes()) + .map_err(|error| error.to_string())?; + + let mut raw = Vec::new(); + let mut chunk = [0_u8; 4096]; + loop { + match stream.read(&mut chunk) { + Ok(0) => break, + Ok(count) => raw.extend_from_slice(&chunk[..count]), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => break, + Err(error) if error.kind() == std::io::ErrorKind::TimedOut => break, + Err(error) => return Err(error.to_string()), + } + } + parse_http(&raw) +} + +fn parse_http(raw: &[u8]) -> Result { + let text = String::from_utf8_lossy(raw); + let Some((head, body)) = text.split_once("\r\n\r\n") else { + return Err(format!("incomplete HTTP response: {text}")); + }; + let mut lines = head.lines(); + let status_line = lines.next().unwrap_or_default(); + let status = status_line + .split_whitespace() + .nth(1) + .and_then(|code| code.parse::().ok()) + .ok_or_else(|| format!("bad status line: {status_line}"))?; + let headers = lines + .filter_map(|line| { + let (name, value) = line.split_once(':')?; + Some((name.trim().to_string(), value.trim().to_string())) + }) + .collect::>(); + let chunked = headers.iter().any(|(name, value)| { + name.eq_ignore_ascii_case("transfer-encoding") + && value.to_ascii_lowercase().contains("chunked") + }); + let body = if chunked { + decode_chunks(body)? + } else { + body.to_string() + }; + Ok(HttpResponse { + status, + headers, + body, + }) +} + +fn decode_chunks(body: &str) -> Result { + let mut rest = body; + let mut out = String::new(); + loop { + let Some((size_line, after)) = rest.split_once("\r\n") else { + return Err(format!("truncated chunk size: {body}")); + }; + let size = usize::from_str_radix(size_line.trim().split(';').next().unwrap_or(""), 16) + .map_err(|error| format!("bad chunk size {size_line}: {error}"))?; + if size == 0 { + return Ok(out); + } + if after.len() < size { + return Err(format!("truncated chunk of {size} bytes")); + } + out.push_str(&after[..size]); + rest = after.get(size + 2..).unwrap_or(""); + } +} diff --git a/crates/tracedecay-mcp/src/handlers/edit.rs b/crates/tracedecay-mcp/src/handlers/edit.rs index cee6bfec9b..68ffab992c 100644 --- a/crates/tracedecay-mcp/src/handlers/edit.rs +++ b/crates/tracedecay-mcp/src/handlers/edit.rs @@ -985,7 +985,7 @@ mod tests { let error = source_edit_refusal(DaemonInvocationOutcome::ApplicationProblem { problem: ApplicationProblem::unavailable( SafeDiagnostic::new( - "application.surface.unavailable", + tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE, "The project runtime for this operation is still mounting", ) .unwrap(), @@ -995,7 +995,10 @@ mod tests { let (reason_code, retryable, _) = error .project_route_context() .expect("warming must stay a typed project-route error"); - assert_eq!(reason_code, "application.surface.unavailable"); + assert_eq!( + reason_code, + tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE + ); assert!(retryable); } diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest.rs index 846387e8ca..00925df9c0 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest.rs @@ -672,30 +672,31 @@ pub async fn ingest_transcript_with_cancellation( route_admission, observations_committed: route_observations_committed, exact_duplicate: route_exact_duplicate, + admission_owns_commit, } = capture; - // Admission is the durable commit; projection is downstream materialization - // off a queue this scope shares with the project catch-up sweep. Counting - // only the projections this pass drained itself reports a pass whose rows a - // peer drainer took as though it had captured nothing. - let authority_changed = messages_upserted > 0 - || route_observations_committed > 0 - || snapshot_capture + let verdict = ingest_commit_verdict(&IngestCommitAccount { + admission_owns_commit, + observations_committed: route_observations_committed, + route_exact_duplicate, + messages_upserted, + snapshot_messages_upserted: snapshot_capture + .as_ref() + .map_or(0, |capture| capture.stats.messages_upserted), + claude_observations_committed: claude_observation_stats + .as_ref() + .map_or(0, |stats| stats.observations_committed), + claude_cursor_advances: claude_observation_stats .as_ref() - .is_some_and(|capture| capture.stats.messages_upserted > 0) - || claude_observation_stats + .map_or(0, |stats| stats.cursor_advances), + claude_observation_duplicates: claude_observation_stats .as_ref() - .is_some_and(|stats| stats.observations_committed > 0 || stats.cursor_advances > 0); - // A pass that changed nothing is only `accepted_for_replay` when it cannot - // prove the data is already there. Routes that can prove it say so: Claude - // through its duplicate counters, every other route through - // `exact_duplicate`. Without this a replay whose observations a peer - // drainer already projected reports a terminal, non-retryable status that - // neither proves a commit nor invites a retry. - let exact_duplicate = !authority_changed - && (route_exact_duplicate - || claude_observation_stats.as_ref().is_some_and(|stats| { - stats.observation_duplicates > 0 || stats.cursor_duplicates > 0 - })); + .map_or(0, |stats| stats.observation_duplicates), + claude_cursor_duplicates: claude_observation_stats + .as_ref() + .map_or(0, |stats| stats.cursor_duplicates), + }); + let authority_changed = verdict.authority_changed; + let exact_duplicate = verdict.exact_duplicate; let deferred_by_byte_cap = source_deferred || snapshot_capture .as_ref() @@ -779,6 +780,57 @@ pub async fn ingest_transcript_with_cancellation( Ok(output) } +/// The counters a capture route hands the terminal-status assembly. +/// +/// `admission_owns_commit` routes (Cursor, Codex project) already know whether +/// they persisted frames. Their projection drain reads a queue the project +/// catch-up also empties, so `messages_upserted` on those routes is a residual +/// of that queue, not a second copy of the commit. +pub(super) struct IngestCommitAccount { + pub(super) admission_owns_commit: bool, + pub(super) observations_committed: u64, + pub(super) route_exact_duplicate: bool, + pub(super) messages_upserted: u64, + pub(super) snapshot_messages_upserted: u64, + pub(super) claude_observations_committed: u64, + pub(super) claude_cursor_advances: u64, + pub(super) claude_observation_duplicates: u64, + pub(super) claude_cursor_duplicates: u64, +} + +pub(super) struct IngestCommitVerdict { + pub(super) authority_changed: bool, + pub(super) exact_duplicate: bool, +} + +/// Commit status from the route that owns it. +/// +/// When admission owns the commit, a non-zero drain residual cannot promote a +/// pass that persisted nothing into `committed`, and a zero drain cannot hide +/// frames this pass did persist. Routes without an admission tally still read +/// their own message and duplicate counters. +pub(super) fn ingest_commit_verdict(account: &IngestCommitAccount) -> IngestCommitVerdict { + if account.admission_owns_commit { + let authority_changed = account.observations_committed > 0; + return IngestCommitVerdict { + authority_changed, + exact_duplicate: !authority_changed && account.route_exact_duplicate, + }; + } + let authority_changed = account.messages_upserted > 0 + || account.observations_committed > 0 + || account.snapshot_messages_upserted > 0 + || account.claude_observations_committed > 0 + || account.claude_cursor_advances > 0; + IngestCommitVerdict { + authority_changed, + exact_duplicate: !authority_changed + && (account.route_exact_duplicate + || account.claude_observation_duplicates > 0 + || account.claude_cursor_duplicates > 0), + } +} + pub(super) fn complete_ingest_admission( admission: HostAdmissionOutcome, authority_changed: bool, diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest/kernels.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest/kernels.rs index fbf97ef025..245e6f64ba 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest/kernels.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest/kernels.rs @@ -125,6 +125,11 @@ pub(super) struct TranscriptCaptureOutcome { /// `messages_upserted` counts only the projections this pass drained /// itself, which a peer drainer can legitimately take first. pub(super) observations_committed: u64, + /// This route's admission tally is the commit. The projection drain is a + /// shared per-scope queue, so its residual must not enter the terminal + /// status. Routes that have no admission tally leave this false and keep + /// using their own message counts. + pub(super) admission_owns_commit: bool, /// The route committed nothing because its observations were already /// durable. Kept apart from `messages_upserted == 0`, which cannot tell an /// already-committed replay from a pass that captured nothing. @@ -429,6 +434,7 @@ async fn capture_codex_project( source_deferred: admitted.deferred, observations_committed: admitted.observations_committed, exact_duplicate: admitted.exact_duplicate, + admission_owns_commit: true, ..TranscriptCaptureOutcome::default() }) } @@ -457,6 +463,7 @@ fn cursor_capture_outcome( source_deferred: stats.source_deferred, observations_committed: stats.observations_committed, exact_duplicate: stats.exact_duplicate, + admission_owns_commit: true, ..TranscriptCaptureOutcome::default() } } diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest/tests.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest/tests.rs index c62091c5d4..9963dd2df8 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest/tests.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/ingest/tests.rs @@ -1,9 +1,99 @@ use super::super::*; use crate::structured_hook_error_data; use tracedecay_project::test_support::host_admission::HostAdmissionTestRuntimeV1; +use tracedecay_sessions::admission::{HostAdmissionOutcome, HostAdmissionStatus}; use super::*; +fn status_for(account: &IngestCommitAccount) -> HostAdmissionStatus { + let verdict = ingest_commit_verdict(account); + complete_ingest_admission( + HostAdmissionOutcome::accepted_for_replay(), + verdict.authority_changed, + verdict.exact_duplicate, + false, + ) + .status +} + +/// A shared-queue residual is not this pass's commit. Nine projected rows +/// left by a peer, or by another provider on the same scope queue, must stay +/// `accepted_for_replay` when admission persisted nothing and cannot prove a +/// duplicate. +#[test] +fn drain_residual_does_not_commit_an_admission_owned_pass() { + let status = status_for(&IngestCommitAccount { + admission_owns_commit: true, + observations_committed: 0, + route_exact_duplicate: false, + messages_upserted: 9, + snapshot_messages_upserted: 0, + claude_observations_committed: 0, + claude_cursor_advances: 0, + claude_observation_duplicates: 0, + claude_cursor_duplicates: 0, + }); + + assert_eq!(status, HostAdmissionStatus::AcceptedForReplay); +} + +/// The pass persisted two observations and the drain found nothing. The +/// commit still stands. +#[test] +fn admission_commit_stands_when_the_drain_is_empty() { + let status = status_for(&IngestCommitAccount { + admission_owns_commit: true, + observations_committed: 2, + route_exact_duplicate: false, + messages_upserted: 0, + snapshot_messages_upserted: 4, + claude_observations_committed: 1, + claude_cursor_advances: 1, + claude_observation_duplicates: 0, + claude_cursor_duplicates: 0, + }); + + assert_eq!(status, HostAdmissionStatus::Committed); +} + +/// A peer already admitted the source. Residual projected rows must not +/// rewrite that duplicate into a fresh commit. +#[test] +fn drain_residual_does_not_promote_an_exact_duplicate() { + let status = status_for(&IngestCommitAccount { + admission_owns_commit: true, + observations_committed: 0, + route_exact_duplicate: true, + messages_upserted: 3, + snapshot_messages_upserted: 0, + claude_observations_committed: 0, + claude_cursor_advances: 0, + claude_observation_duplicates: 0, + claude_cursor_duplicates: 0, + }); + + assert_eq!(status, HostAdmissionStatus::ExactDuplicate); +} + +/// Hermes and the other routes that have no admission tally still commit +/// from the messages they themselves upserted. +#[test] +fn message_counted_route_still_commits_from_its_own_upserts() { + let status = status_for(&IngestCommitAccount { + admission_owns_commit: false, + observations_committed: 0, + route_exact_duplicate: false, + messages_upserted: 1, + snapshot_messages_upserted: 0, + claude_observations_committed: 0, + claude_cursor_advances: 0, + claude_observation_duplicates: 0, + claude_cursor_duplicates: 0, + }); + + assert_eq!(status, HostAdmissionStatus::Committed); +} + #[test] fn cursor_compaction_response_matches_hook_contract() { let value = cursor_compact_skipped("no messages to compact"); diff --git a/crates/tracedecay-mcp/src/server/project_host_admission_replay.rs b/crates/tracedecay-mcp/src/server/project_host_admission_replay.rs index 6d4f8c5a1d..851e75b05a 100644 --- a/crates/tracedecay-mcp/src/server/project_host_admission_replay.rs +++ b/crates/tracedecay-mcp/src/server/project_host_admission_replay.rs @@ -203,6 +203,10 @@ impl ProjectHostAdmissionReplayWorker { ); break; } + ReplayPassDecision::TerminalNoop => { + consecutive_retryable = 0; + break; + } ReplayPassDecision::Requeue => { consecutive_retryable = 0; if self.dirty.load(Ordering::Acquire) || pending_after > 0 { @@ -275,6 +279,42 @@ mod tests { task.shutdown().await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn not_applicable_pending_record_stops_without_backoff() { + let temp = tempfile::TempDir::new().unwrap(); + let (runtime, _) = tracedecay_host_admission::HostAdmissionRuntime::open( + temp.path(), + tracedecay_host_admission::SpoolBounds::default(), + ) + .unwrap(); + let broker = Arc::new(tracedecay_host_admission::HostAdmissionBroker::new(runtime)); + broker.admit("test:pending", b"pending").await.unwrap(); + let passes = Arc::new(AtomicUsize::new(0)); + let passes_for_run = Arc::clone(&passes); + let pass: PassFn = Arc::new(move || { + let passes = Arc::clone(&passes_for_run); + Box::pin(async move { + passes.fetch_add(1, Ordering::AcqRel); + HostAdmissionOutcome::not_applicable("code_index_not_applicable") + }) + }); + let task = ProjectHostAdmissionReplayTask::start(broker, pass); + + tokio::time::timeout(Duration::from_secs(1), async { + while task.pass_count() == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("project replay must attempt the not-applicable record"); + tokio::time::sleep(Duration::from_millis(100)).await; + + assert_eq!(passes.load(Ordering::Acquire), 1); + assert_eq!(task.pass_count(), 1); + assert_eq!(task.backoff_count(), 0); + task.shutdown().await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn dropping_task_aborts_an_in_flight_pass_without_an_arc_cycle() { let temp = tempfile::TempDir::new().unwrap(); diff --git a/crates/tracedecay-query/Cargo.toml b/crates/tracedecay-query/Cargo.toml index 015e41e6cb..ffc65866d4 100644 --- a/crates/tracedecay-query/Cargo.toml +++ b/crates/tracedecay-query/Cargo.toml @@ -47,6 +47,8 @@ test-helpers = ["tracedecay-temporal-query/test-helpers"] # CodeLexicalProjectionAdapterV1` and its builder). Production retrieval reads # the durable lexical artifact; only the search-quality evaluator and this # crate's own suites build projections directly from admitted chunks. +# `tracedecay` must not depend on the evaluator unconditionally: that unifies +# this feature into every root test target, including transport. search-eval = [] # The grammar tier the shipped product indexes with, forwarded exactly as # `tracedecay-cli`'s `production` forwards `tracedecay/production`. Only the diff --git a/crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json b/crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json index 4646d75e25..7870e2a69a 100644 --- a/crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json +++ b/crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json @@ -135,8 +135,8 @@ } ], "expected_query_fallback_digests": { - "train": "sha256:5750d4a588f7a7e14c381ec3a4285400a29e164babbf88e677ce7441aaf8e9b2", - "validation": "sha256:e7a459efb1655bb71e30fd302690cd5ac7937197add94a3ac1cab5812a1f5a48" + "train": "sha256:f3744c56a8d35a13ddd9616eb949ae31b2bf745bf14b4d0bdc97c3e0ed139660", + "validation": "sha256:561eb59d2b58d5c935b28d0217dc18252a2d2c28e47f404cc59104e03c95625d" }, "profile_matrix": [ { diff --git a/crates/tracedecay-query/src/search_quality/candidate_output.rs b/crates/tracedecay-query/src/search_quality/candidate_output.rs index bd62d10937..e329d77b26 100644 --- a/crates/tracedecay-query/src/search_quality/candidate_output.rs +++ b/crates/tracedecay-query/src/search_quality/candidate_output.rs @@ -72,6 +72,13 @@ pub struct CandidateWorkloadV1 { pub execution_contract: EvaluationExecutionContractV1, pub corpus: Vec, pub profile_matrix: Vec, + /// Observed ranking receipts for `train` and `validation`. + /// + /// These bind ordered ranking rows and public lane coverage. They are not + /// workload inputs: [`compute_workload_digest`] omits them, so re-pinning a + /// receipt does not rewrite the packaged workload identity. Generation and + /// extractor-revision changes reseal candidate occurrence ids without + /// changing those rows, and must not move this field either. pub expected_query_fallback_digests: BTreeMap, pub queries: Vec, } @@ -393,7 +400,12 @@ pub fn load_candidate_workload(path: &Path) -> Result Result { - canonical_sha256(workload) + // Ranking receipts observe the run. Including them made a receipt re-pin + // look like a different workload, including when only a sealed generation + // id moved. + let mut identity = workload.clone(); + identity.expected_query_fallback_digests.clear(); + canonical_sha256(&("tracedecay.search-eval.workload-identity.v1", &identity)) } pub fn compute_profile_material_digest( @@ -1014,6 +1026,24 @@ mod need_provenance_tests { ); } + #[test] + fn ranking_receipt_edits_do_not_move_workload_identity() { + let workload = workload(); + let identity = super::compute_workload_digest(&workload).expect("workload identity"); + assert_eq!(identity, packaged::WORKLOAD_SHA256); + let mut moved = workload; + let train = moved + .expected_query_fallback_digests + .get_mut("train") + .expect("train receipt"); + *train = format!("sha256:{}", "ab".repeat(32)); + assert_eq!( + super::compute_workload_digest(&moved).expect("moved identity"), + identity, + "re-pinning a ranking receipt must not rewrite the workload identity" + ); + } + #[test] fn a_need_without_documented_provenance_is_refused() { let mut workload = workload(); diff --git a/crates/tracedecay-query/src/search_quality/packaged.rs b/crates/tracedecay-query/src/search_quality/packaged.rs index 4b3556a777..1417d88239 100644 --- a/crates/tracedecay-query/src/search_quality/packaged.rs +++ b/crates/tracedecay-query/src/search_quality/packaged.rs @@ -1,13 +1,19 @@ -use tracedecay_domain::canonical_text::sha256_hex; - use super::candidate_output::{ - CandidateWorkloadV1, validate_need_provenance_against_embedded_corpus, + CandidateWorkloadV1, compute_workload_digest, validate_need_provenance_against_embedded_corpus, validate_workload_for_tuning, }; use super::evaluate::SearchEvalError; const WORKLOAD_PATH: &str = "tests/fixtures/search_quality/query-lexical-graph-workload-v1.json"; -const WORKLOAD_SHA256: &str = "267e2bd2e9b90d258cbeed829920ab735f6af0ebc2a6e870d59eeef29b1cdb93"; +/// Canonical identity of the packaged workload inputs. +/// +/// This is [`compute_workload_digest`]: schema, queries, corpus, profile, and +/// execution contract. It deliberately excludes `expected_query_fallback_digests`. +/// Those receipts observe ranking; folding them into this pin made every +/// receipt edit, including a generation-only reseal, rewrite the workload +/// identity as well. +pub const WORKLOAD_SHA256: &str = + "sha256:883b1dc8673f0bdf09f54fd4e4bc598e7df01607933a6bf4053f31003b111d31"; const FILES: &[(&str, &[u8])] = &[ ( @@ -110,15 +116,17 @@ pub fn packaged_evaluator_files() -> &'static [(&'static str, &'static [u8])] { #[hotpath::measure(label = "search_eval.packaged.load_workload")] pub fn load_workload() -> Result { - let observed_workload_digest = sha256_hex(FILES[0].1); + let workload = serde_json::from_slice::(FILES[0].1).map_err(|error| { + SearchEvalError::Contract(format!("parse packaged evaluator workload: {error}")) + })?; + let observed_workload_digest = compute_workload_digest(&workload).map_err(|error| { + SearchEvalError::Contract(format!("hash packaged evaluator workload: {error}")) + })?; if observed_workload_digest != WORKLOAD_SHA256 { return Err(SearchEvalError::Contract(format!( "packaged evaluator workload digest mismatch: expected {WORKLOAD_SHA256}, observed {observed_workload_digest}" ))); } - let workload = serde_json::from_slice::(FILES[0].1).map_err(|error| { - SearchEvalError::Contract(format!("parse packaged evaluator workload: {error}")) - })?; validate_workload_for_tuning(&workload)?; validate_need_provenance_against_embedded_corpus(&workload, FILES)?; Ok(workload) diff --git a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs index e006138f83..9009f6ecdd 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs @@ -1453,6 +1453,15 @@ fn v16_clone_payloads_are_content_addressed_and_postings_page() { .append_page(&pages[0], &control) .expect("append first clone page"); drop(successor); + // A successor staged before occurrence indexes existed must still verify + // on resume. Dropping them here is that shipped shape. + rusqlite::Connection::open(&successor_path) + .expect("open successor before index backfill") + .execute_batch( + "DROP INDEX IF EXISTS clone_exact_postings_by_occurrence; + DROP INDEX IF EXISTS clone_fingerprint_postings_by_occurrence;", + ) + .expect("drop occurrence indexes"); let mut successor = CodeLexicalCloneSuccessorV1::open_or_create( &legacy_path, &successor_path, @@ -1461,6 +1470,23 @@ fn v16_clone_payloads_are_content_addressed_and_postings_page() { CODE_LEXICAL_ARTIFACT_BUILD_MEMORY_BUDGET_BYTES_V1, ) .expect("resume clone-only successor"); + // The resume reads the same rows with or without the indexes, so assert + // the backfill itself as well as the verification it is there to speed up. + let backfilled = rusqlite::Connection::open(&successor_path) + .expect("open successor after index backfill") + .query_row( + "SELECT count(*) FROM sqlite_master WHERE type = 'index' AND name IN ('clone_exact_postings_by_occurrence', 'clone_fingerprint_postings_by_occurrence')", + [], + |row| row.get::<_, i64>(0), + ) + .expect("count occurrence indexes"); + assert_eq!( + backfilled, 2, + "opening a successor staged before the occurrence indexes must install both" + ); + successor + .verify_resumed_page(&pages[0], &control) + .expect("resumed clone page verifies through the occurrence index"); assert_eq!( successor .next_cursor() diff --git a/crates/tracedecay-runtime-core/src/resident_memory.rs b/crates/tracedecay-runtime-core/src/resident_memory.rs index f1669a488a..a40c4d5b50 100644 --- a/crates/tracedecay-runtime-core/src/resident_memory.rs +++ b/crates/tracedecay-runtime-core/src/resident_memory.rs @@ -23,8 +23,8 @@ pub const DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1: NonZeroU64 = /// Environment override for the process resident-memory admission limit, in /// bytes. Unset, unparseable, or zero values fall back to the RAM-derived /// authority. The code-index worker pool derives its reservation from this -/// same limit, so raising it can both admit and widen indexing, up to any -/// finite cgroup-v2 memory ceiling. +/// same limit, so raising it can both admit and widen indexing, up to the +/// hard cgroup ceiling (`memory.max`, or `memory.high` when max is unlimited). pub const PROCESS_RESIDENT_MEMORY_LIMIT_ENV_V1: &str = "TRACEDECAY_RESIDENT_MEMORY_LIMIT_BYTES"; const PROC_SELF_CGROUP_V1: &str = "/proc/self/cgroup"; @@ -85,15 +85,41 @@ fn finite_cgroup_memory_value_v1(path: &Path) -> Option { value.parse::().ok().map(|value| value.max(1)) } -fn cgroup_v2_memory_limit_v1(proc_self_cgroup: &Path, cgroup_root: &Path) -> Option { +/// The two cgroup-v2 memory controls on this process, walked to the mount root. +/// +/// `memory.max` is the kernel kill line. `memory.high` is the reclaim line +/// underneath it. They stay separate so each is used for what it is. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct CgroupMemoryCeilingV1 { + max_bytes: Option, + high_bytes: Option, +} + +/// Hard service ceiling: `memory.max` when it is finite, otherwise `memory.high`. +/// +/// A lone `memory.high` is the ceiling because the operator left no band above +/// the reclaim line. When both are finite, high is pressure, not a tighter max. +fn cgroup_service_ceiling_bytes(ceiling: CgroupMemoryCeilingV1) -> Option { + ceiling.max_bytes.or(ceiling.high_bytes) +} + +fn tighten(bound: Option, limit: u64) -> u64 { + bound.map_or(limit, |current| current.min(limit)) +} + +fn cgroup_v2_memory_ceiling_v1( + proc_self_cgroup: &Path, + cgroup_root: &Path, +) -> Option { let mut directory = cgroup_v2_process_directory_v1(proc_self_cgroup, cgroup_root)?; - let mut effective_limit = None; + let mut max_bytes = None; + let mut high_bytes = None; loop { - for filename in ["memory.max", "memory.high"] { - if let Some(limit) = finite_cgroup_memory_value_v1(&directory.join(filename)) { - effective_limit = - Some(effective_limit.map_or(limit, |current: u64| current.min(limit))); - } + if let Some(limit) = finite_cgroup_memory_value_v1(&directory.join("memory.max")) { + max_bytes = Some(tighten(max_bytes, limit)); + } + if let Some(limit) = finite_cgroup_memory_value_v1(&directory.join("memory.high")) { + high_bytes = Some(tighten(high_bytes, limit)); } if directory == cgroup_root { break; @@ -104,7 +130,10 @@ fn cgroup_v2_memory_limit_v1(proc_self_cgroup: &Path, cgroup_root: &Path) -> Opt } directory = parent.to_path_buf(); } - effective_limit + Some(CgroupMemoryCeilingV1 { + max_bytes, + high_bytes, + }) } fn effective_memory_bytes_v1(total_memory_bytes: u64, cgroup_limit: Option) -> u64 { @@ -115,68 +144,99 @@ fn effective_memory_bytes_v1(total_memory_bytes: u64, cgroup_limit: Option) } } -fn process_resident_memory_limit_v1( +struct ResidentMemoryAuthorityV1 { + limit_bytes: NonZeroU64, + /// `memory.high` when it sits strictly below the hard admission ceiling. + reclaim_watermark_bytes: Option, +} + +fn finite_nonzero_bytes(value: u64) -> NonZeroU64 { + NonZeroU64::new(value).unwrap_or(DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1) +} + +/// Admission ceiling for one host and one cgroup reading. +/// +/// Host reserve (one quarter of physical RAM) and the cgroup service ceiling +/// are alternative protections, not stacked discounts. The reserve applies +/// when the process can otherwise spend the machine. A finite cgroup already +/// reserved the rest of the machine, so the hard ceiling is +/// `min(host allowance, memory.max)` — or `memory.high` only when max is +/// unlimited. `memory.high` below that ceiling is the reclaim watermark, not +/// a second cut. An explicit override replaces the host reserve and is still +/// capped by the hard ceiling. +fn resident_memory_authority_v1( total_memory_bytes: u64, - cgroup_limit: Option, + cgroup: Option, override_limit: Option, -) -> NonZeroU64 { - // Retain one quarter of physical RAM for the OS and other processes, then - // respect the operator's cgroup ceiling as-is. Applying the quarter again - // *after* taking min(host, cgroup) double-discounted a deliberately sized - // service: 128 GiB host, memory.high=26 GiB became 19.5 GiB even though - // memory.max=30 GiB already retained the safety margin. An 18 GiB serving - // graph could then never admit its 2.6 GiB replacement builder. - let host_limit = (total_memory_bytes != 0) +) -> ResidentMemoryAuthorityV1 { + let cgroup = cgroup.unwrap_or(CgroupMemoryCeilingV1 { + max_bytes: None, + high_bytes: None, + }); + let service_ceiling = cgroup_service_ceiling_bytes(cgroup); + let host_allowance = (total_memory_bytes != 0) .then(|| process_resident_memory_limit_for_system_v1(total_memory_bytes)); - let automatic_limit = match (host_limit, cgroup_limit) { - (Some(host), Some(cgroup)) => NonZeroU64::new(host.get().min(cgroup)) - .unwrap_or(DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1), + let automatic_limit = match (host_allowance, service_ceiling) { + (Some(host), Some(ceiling)) => finite_nonzero_bytes(host.get().min(ceiling)), (Some(host), None) => host, - (None, Some(cgroup)) => { - NonZeroU64::new(cgroup).unwrap_or(DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1) - } + (None, Some(ceiling)) => finite_nonzero_bytes(ceiling), (None, None) => DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1, }; - override_limit.map_or(automatic_limit, |override_limit| { - cgroup_limit.map_or(override_limit, |cgroup_limit| { - NonZeroU64::new(override_limit.get().min(cgroup_limit)) - .unwrap_or(DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1) - }) - }) + let limit_bytes = match override_limit { + Some(override_limit) => match service_ceiling { + Some(ceiling) => finite_nonzero_bytes(override_limit.get().min(ceiling)), + None => override_limit, + }, + None => automatic_limit, + }; + let reclaim_watermark_bytes = cgroup.high_bytes.filter(|high| *high < limit_bytes.get()); + ResidentMemoryAuthorityV1 { + limit_bytes, + reclaim_watermark_bytes, + } } /// Size the shared resident-allocation authority for this process. /// -/// The automatic authority retains one quarter of physical RAM, then takes the -/// lower of that host allowance and this process's finite cgroup-v2 -/// `memory.max` / `memory.high`. A cgroup is already an operator-sized service -/// allowance and is not discounted a second time. +/// The automatic authority is the lower of the host reserve and this +/// process's hard cgroup ceiling (`memory.max`, or `memory.high` when max is +/// unlimited). A finite `memory.high` below that ceiling is the pressure +/// watermark, not a further discount of the ceiling. /// [`PROCESS_RESIDENT_MEMORY_LIMIT_ENV_V1`] can lower or raise the automatic -/// authority, but a finite cgroup ceiling remains an upper bound. The resulting -/// authority throttles simultaneous scratch ownership; it never limits -/// repository bytes on disk. +/// authority, but the hard cgroup ceiling remains an upper bound. The +/// resulting authority throttles simultaneous scratch ownership; it never +/// limits repository bytes on disk. #[must_use] pub fn detected_process_resident_memory_limit_v1() -> NonZeroU64 { + read_resident_memory_authority_v1().limit_bytes +} + +fn read_resident_memory_authority_v1() -> ResidentMemoryAuthorityV1 { let system = System::new_with_specifics( RefreshKind::new().with_memory(MemoryRefreshKind::new().with_ram()), ); let total_memory_bytes = system.total_memory(); let proc_self_cgroup = Path::new(PROC_SELF_CGROUP_V1); let cgroup_root = Path::new(CGROUP_V2_ROOT_V1); - let cgroup_limit = cgroup_v2_memory_limit_v1(proc_self_cgroup, cgroup_root); - let effective_memory_bytes = effective_memory_bytes_v1(total_memory_bytes, cgroup_limit); - let limit = process_resident_memory_limit_v1( + let cgroup = cgroup_v2_memory_ceiling_v1(proc_self_cgroup, cgroup_root); + let service_ceiling = cgroup.and_then(cgroup_service_ceiling_bytes); + let effective_memory_bytes = effective_memory_bytes_v1(total_memory_bytes, service_ceiling); + let authority = resident_memory_authority_v1( total_memory_bytes, - cgroup_limit, + cgroup, process_resident_memory_limit_override_v1(), ); hotpath::gauge!("resident_memory.system_total_bytes").set(total_memory_bytes as f64); hotpath::gauge!("resident_memory.effective_total_bytes").set(effective_memory_bytes as f64); - if let Some(cgroup_limit) = cgroup_limit { - hotpath::gauge!("resident_memory.cgroup_limit_bytes").set(cgroup_limit as f64); + if let Some(high_bytes) = cgroup.and_then(|ceiling| ceiling.high_bytes) { + hotpath::gauge!("resident_memory.cgroup_high_bytes").set(high_bytes as f64); + } + if let Some(service_ceiling) = service_ceiling { + hotpath::gauge!("resident_memory.cgroup_limit_bytes").set(service_ceiling as f64); } - hotpath::gauge!("resident_memory.admission_limit_bytes").set(limit.get() as f64); - limit + hotpath::gauge!("resident_memory.admission_limit_bytes") + .set(authority.limit_bytes.get() as f64); + authority } /// Fraction of the configured limit, in permille, at or above which *measured* @@ -343,15 +403,36 @@ impl fmt::Debug for ResidentMemoryPressureV1 { impl ResidentMemoryPressureV1 { #[must_use] pub fn new(limit_bytes: NonZeroU64) -> Self { - let high_watermark_bytes = resident_memory_watermark_bytes_v1( + Self::with_reclaim_line(limit_bytes, None) + } + + /// `reclaim_watermark_bytes` is a cgroup `memory.high` that sits strictly + /// below `limit_bytes`. It replaces the percentage high watermark so the + /// operator's band down to `memory.max` is not discounted again. Absent, + /// zero, or not strictly below the ceiling, the percentage watermarks stand. + fn with_reclaim_line(limit_bytes: NonZeroU64, reclaim_watermark_bytes: Option) -> Self { + let percentage_high = resident_memory_watermark_bytes_v1( limit_bytes, RESIDENT_MEMORY_PRESSURE_HIGH_WATERMARK_PERMILLE_V1, ); - let low_watermark_bytes = resident_memory_watermark_bytes_v1( + let percentage_low = resident_memory_watermark_bytes_v1( limit_bytes, RESIDENT_MEMORY_PRESSURE_LOW_WATERMARK_PERMILLE_V1, ) - .min(high_watermark_bytes); + .min(percentage_high); + let (high_watermark_bytes, low_watermark_bytes) = match reclaim_watermark_bytes { + Some(reclaim) if reclaim > 0 && reclaim < limit_bytes.get() => { + let low = u64::try_from( + u128::from(reclaim) + * u128::from(RESIDENT_MEMORY_PRESSURE_LOW_WATERMARK_PERMILLE_V1) + / u128::from(RESIDENT_MEMORY_PRESSURE_HIGH_WATERMARK_PERMILLE_V1), + ) + .unwrap_or(u64::MAX) + .min(reclaim); + (reclaim, low) + } + _ => (percentage_high, percentage_low), + }; Self { limit_bytes, high_watermark_bytes, @@ -547,8 +628,10 @@ static PROCESS_RESIDENT_MEMORY_PRESSURE_V1: OnceLock &'static Arc { PROCESS_RESIDENT_MEMORY_PRESSURE_V1.get_or_init(|| { - Arc::new(ResidentMemoryPressureV1::new( - detected_process_resident_memory_limit_v1(), + let authority = read_resident_memory_authority_v1(); + Arc::new(ResidentMemoryPressureV1::with_reclaim_line( + authority.limit_bytes, + authority.reclaim_watermark_bytes, )) }) } diff --git a/crates/tracedecay-runtime-core/src/resident_memory/tests.rs b/crates/tracedecay-runtime-core/src/resident_memory/tests.rs index 53bc42ca60..292ad8f2cd 100644 --- a/crates/tracedecay-runtime-core/src/resident_memory/tests.rs +++ b/crates/tracedecay-runtime-core/src/resident_memory/tests.rs @@ -5,10 +5,11 @@ use std::sync::{Arc, Mutex, OnceLock}; use tracedecay_domain::{CodeGenerationId, ProjectId, WorktreeId}; use super::{ - ProcessResidentMemoryV1, RESIDENT_MEMORY_PRESSURE_ADMISSION_FLOOR_BYTES_V1, - ResidentMemoryAdmissionFailureV1, ResidentMemoryComponentIdV1, ResidentMemoryKeyV1, - ResidentMemoryPressureStateV1, ResidentMemoryPressureV1, cgroup_v2_memory_limit_v1, - effective_memory_bytes_v1, process_resident_memory_limit_v1, + CgroupMemoryCeilingV1, ProcessResidentMemoryV1, + RESIDENT_MEMORY_PRESSURE_ADMISSION_FLOOR_BYTES_V1, ResidentMemoryAdmissionFailureV1, + ResidentMemoryComponentIdV1, ResidentMemoryKeyV1, ResidentMemoryPressureStateV1, + ResidentMemoryPressureV1, cgroup_service_ceiling_bytes, cgroup_v2_memory_ceiling_v1, + effective_memory_bytes_v1, resident_memory_authority_v1, }; fn bytes(value: u64) -> NonZeroU64 { @@ -45,7 +46,8 @@ fn effective_memory_bytes( ) -> u64 { effective_memory_bytes_v1( total_memory_bytes, - cgroup_v2_memory_limit_v1(proc_self_cgroup, cgroup_root), + cgroup_v2_memory_ceiling_v1(proc_self_cgroup, cgroup_root) + .and_then(cgroup_service_ceiling_bytes), ) } @@ -115,30 +117,58 @@ fn root_v2_membership_reads_the_mount_root_ceiling() { ); } +fn hard_ceiling(max_bytes: u64) -> CgroupMemoryCeilingV1 { + CgroupMemoryCeilingV1 { + max_bytes: Some(max_bytes), + high_bytes: None, + } +} + #[test] fn configured_override_cannot_exceed_the_cgroup_ceiling() { let gib = 1024 * 1024 * 1024; + let capped = resident_memory_authority_v1( + 88 * gib, + Some(CgroupMemoryCeilingV1 { + max_bytes: Some(30 * gib), + high_bytes: Some(26 * gib), + }), + Some(bytes(64 * gib)), + ); assert_eq!( - process_resident_memory_limit_v1(88 * gib, Some(30 * gib), Some(bytes(64 * gib))).get(), - 30 * gib + capped.limit_bytes.get(), + 30 * gib, + "an override is capped by memory.max, not by the reclaim line" ); + assert_eq!(capped.reclaim_watermark_bytes, Some(26 * gib)); } #[test] fn cgroup_service_allowance_is_not_discounted_twice() { let gib = 1024 * 1024 * 1024; - + let only_high = resident_memory_authority_v1( + 128 * gib, + Some(CgroupMemoryCeilingV1 { + max_bytes: None, + high_bytes: Some(26 * gib), + }), + None, + ); assert_eq!( - process_resident_memory_limit_v1(128 * gib, Some(26 * gib), None).get(), + only_high.limit_bytes.get(), 26 * gib, - "the cgroup already reserves host headroom for this service" + "a lone memory.high is the service ceiling and is not quartered again" ); + assert_eq!(only_high.reclaim_watermark_bytes, None); + + let small_host = resident_memory_authority_v1(16 * gib, Some(hard_ceiling(30 * gib)), None); assert_eq!( - process_resident_memory_limit_v1(16 * gib, Some(30 * gib), None).get(), + small_host.limit_bytes.get(), 12 * gib, "a larger cgroup must not erase the physical-host reserve" ); + assert_eq!(small_host.reclaim_watermark_bytes, None); } #[test] @@ -155,7 +185,7 @@ fn unlimited_cgroup_memory_files_keep_host_memory_capacity() { } #[test] -fn finite_memory_high_below_max_is_the_effective_capacity() { +fn memory_high_does_not_replace_memory_max_as_the_hard_capacity() { let gib = 1024 * 1024 * 1024; let (_directory, proc_self_cgroup, cgroup_root) = cgroup_fixture( Some("0::/trace.slice/daemon.scope\n"), @@ -164,8 +194,13 @@ fn finite_memory_high_below_max_is_the_effective_capacity() { ); assert_eq!( effective_memory_bytes(88 * gib, &proc_self_cgroup, &cgroup_root), - 24 * gib + 30 * gib, + "memory.max is the kernel kill line" ); + let ceiling = cgroup_v2_memory_ceiling_v1(&proc_self_cgroup, &cgroup_root).expect("cgroup"); + let authority = resident_memory_authority_v1(88 * gib, Some(ceiling), None); + assert_eq!(authority.limit_bytes.get(), 30 * gib); + assert_eq!(authority.reclaim_watermark_bytes, Some(24 * gib)); } #[test] @@ -188,6 +223,89 @@ fn finite_ancestor_limit_bounds_an_unlimited_process_cgroup() { drop(directory); } +/// The slice owns `memory.max` and the service owns `memory.high`. +/// +/// On a 128 GiB host those are 30 GiB and 26 GiB. RSS at 24 GiB is still under +/// the reclaim line, so the authority admits growth instead of latching at a +/// percentage of a ceiling the operator never set. +#[test] +fn slice_max_and_service_high_keep_the_reclaim_band_usable() { + let gib = 1024 * 1024 * 1024; + let (directory, proc_self_cgroup, cgroup_root) = cgroup_fixture( + Some("0::/trace.slice/daemon.scope\n"), + Some("max\n"), + Some(&format!("{}\n", 26 * gib)), + ); + fs::write( + cgroup_root.join("trace.slice/memory.max"), + format!("{}\n", 30 * gib), + ) + .expect("ancestor memory.max fixture"); + fs::write(cgroup_root.join("trace.slice/memory.high"), "max\n") + .expect("ancestor memory.high fixture"); + + let ceiling = cgroup_v2_memory_ceiling_v1(&proc_self_cgroup, &cgroup_root).expect("cgroup"); + let detected = resident_memory_authority_v1(128 * gib, Some(ceiling), None); + let pressure = Arc::new(ResidentMemoryPressureV1::with_reclaim_line( + detected.limit_bytes, + detected.reclaim_watermark_bytes, + )); + let authority = Arc::new(ProcessResidentMemoryV1::with_pressure( + detected.limit_bytes, + Arc::clone(&pressure), + )); + + assert_eq!(detected.limit_bytes.get(), 30 * gib); + assert_eq!(pressure.high_watermark_bytes(), 26 * gib); + assert_eq!( + pressure.low_watermark_bytes(), + 23_264_406_186, + "hysteresis stays at 750/900 of memory.high, not 75% of memory.max" + ); + assert!( + !pressure + .publish_observed_resident_bytes(22 * gib) + .is_over_budget() + ); + assert!( + !pressure + .publish_observed_resident_bytes(24 * gib) + .is_over_budget(), + "rss under memory.high is not over budget" + ); + authority + .reserve( + key("project-a", "worktree-a", "generation-a", "text-build"), + bytes(3 * gib), + ) + .expect("the process authority admits a 3 GiB reservation at 24 GiB RSS"); + + // Text-artifact admission spends the band down to the reclaim line, never + // down to memory.max: `text_artifact_admitted_build_budget` subtracts the + // same watermark headroom it charges, so its growth budget reduces to + // `high_watermark - observed`. At 24 GiB observed that is 2 GiB, clearing + // the 1536 MiB builder floor. The 90%-of-26 GiB watermark left 0 and + // deadlocked the replacement build. + let headroom = detected + .limit_bytes + .get() + .saturating_sub(pressure.high_watermark_bytes()); + let available_for_growth = detected + .limit_bytes + .get() + .saturating_sub(24 * gib) + .saturating_sub(headroom); + assert_eq!(available_for_growth, 2 * gib); + assert!(available_for_growth >= 1536 * 1024 * 1024); + + assert!( + pressure + .publish_observed_resident_bytes(26 * gib) + .is_over_budget() + ); + drop(directory); +} + #[test] fn low_effective_cgroup_ceiling_engages_measured_pressure_before_the_cap() { let mib = 1024 * 1024; @@ -196,17 +314,26 @@ fn low_effective_cgroup_ceiling_engages_measured_pressure_before_the_cap() { Some("134217728\n"), Some("100663296\n"), ); - let effective = effective_memory_bytes(8 * 1024 * mib, &proc_self_cgroup, &cgroup_root); - let limit = process_resident_memory_limit_v1(8 * 1024 * mib, Some(effective), None); - let pressure = Arc::new(ResidentMemoryPressureV1::new(limit)); + let ceiling = cgroup_v2_memory_ceiling_v1(&proc_self_cgroup, &cgroup_root).expect("cgroup"); + let detected = resident_memory_authority_v1(8 * 1024 * mib, Some(ceiling), None); + let pressure = Arc::new(ResidentMemoryPressureV1::with_reclaim_line( + detected.limit_bytes, + detected.reclaim_watermark_bytes, + )); let authority = Arc::new(ProcessResidentMemoryV1::with_pressure( - limit, + detected.limit_bytes, Arc::clone(&pressure), )); - assert_eq!(effective, 96 * mib); - assert_eq!(limit.get(), effective); - assert!(pressure.high_watermark_bytes() < effective); + assert_eq!(detected.limit_bytes.get(), 128 * mib); + assert_eq!(pressure.high_watermark_bytes(), 96 * mib); + assert!(pressure.high_watermark_bytes() < detected.limit_bytes.get()); + assert!( + !pressure + .publish_observed_resident_bytes(95 * mib) + .is_over_budget(), + "rss below memory.high is still under the hard ceiling" + ); assert!( pressure .publish_observed_resident_bytes(pressure.high_watermark_bytes()) diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/observation/mod.rs b/crates/tracedecay-rusqlite-runtime/src/repository/observation/mod.rs index 1742ad8d6b..0ff4734612 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/observation/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/observation/mod.rs @@ -177,16 +177,16 @@ impl ObservationExecutor { let source_json = encode(advance.next_cursor().source())?; let scope_json = encode(advance.next_cursor().scope())?; let actual_cursor = read_cursor(savepoint, &source_json, &scope_json)?; - if actual_cursor.as_ref() == Some(advance.next_cursor()) { - if let Some(disagreement) = - cursor_advance_ledger_disagreement(savepoint, &source_json, &scope_json, advance)? - { - return Err(disagreement); - } - if cursor_advance_receipt_matches(savepoint, &source_json, &scope_json, advance)? { - return Ok(()); - } - return Err(StorageOperationError::ObservationCursorAdvanceCollision); + // The durable cursor owns the range. Live ingest and catch-up both + // advance the same bytes with legitimately different reasons; once + // the frontier is reached the first ledger row stays and the later + // owner is a no-op. A disagreement is still a failure below, when + // this advance would be the write that moves the cursor. + if actual_cursor + .as_ref() + .is_some_and(|cursor| cursor.reached(advance.next_cursor())) + { + return Ok(()); } if actual_cursor.as_ref() != advance.expected_cursor() { return Err(observation_source_cursor_conflict( diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/observation/tests.rs b/crates/tracedecay-rusqlite-runtime/src/repository/observation/tests.rs index dd19c6c519..6079a44a5f 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/observation/tests.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/observation/tests.rs @@ -478,6 +478,20 @@ fn execute_cursor_advance( Ok(()) } +fn source_cursor_json(connection: &Connection) -> String { + connection + .query_row("SELECT cursor_json FROM source_cursors", [], |row| { + row.get(0) + }) + .unwrap() +} + +fn restore_source_cursor(connection: &Connection, cursor_json: &str) { + connection + .execute("UPDATE source_cursors SET cursor_json = ?1", [cursor_json]) + .unwrap(); +} + #[test] fn anchored_write_persists_all_authority_rows_atomically() { let mut connection = connection(); @@ -738,10 +752,11 @@ fn identity_collision_fails_without_advancing_the_source_cursor() { } #[test] -fn source_cursor_advance_replays_exactly_and_reports_ledger_disagreement() { +fn source_cursor_advance_keeps_the_first_owner_once_the_frontier_is_reached() { let mut connection = connection(); let write = anchored_observation_write("fixture", "receipt.fixture"); execute(&mut connection, &write).unwrap(); + let owned_frontier = source_cursor_json(&connection); let advance = ObservationCursorAdvance::for_ordering( write.observation().source().clone(), write.observation().scope().clone(), @@ -774,9 +789,30 @@ fn source_cursor_advance_replays_exactly_and_reports_ledger_disagreement() { ObservationCoverageReason::OutOfScope, ) .unwrap(); + execute_cursor_advance(&mut connection, &conflicting).unwrap(); + assert_eq!( + connection + .query_row("SELECT reason FROM source_cursor_advances", [], |row| { + row.get::<_, String>(0) + },) + .unwrap(), + ObservationCoverageReason::BlankFrame.as_str() + ); + assert_eq!( + connection + .query_row("SELECT COUNT(*) FROM source_cursor_advances", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 1 + ); + + // The disagreement stays a write-time failure: the cursor has not + // reached the proposed frontier, so this advance would move it. + restore_source_cursor(&connection, &owned_frontier); let error = execute_cursor_advance(&mut connection, &conflicting).unwrap_err(); let StorageOperationError::CursorAdvanceLedgerDisagreement { disagreement } = error else { - panic!("expected structured immutable ledger disagreement"); + panic!("expected structured immutable ledger disagreement, got {error:?}"); }; assert_eq!(disagreement.source(), write.observation().source()); assert_eq!(disagreement.scope(), write.observation().scope()); @@ -804,6 +840,7 @@ fn canonical_cursor_advance_receipt_remains_typed_after_authority_lookup() { let mut connection = connection(); let write = anchored_observation_write("fixture", "receipt.fixture"); execute(&mut connection, &write).unwrap(); + let observation_cursor = source_cursor_json(&connection); let advance = ObservationCursorAdvance::for_ordering_with_sanitization_receipt( write.observation().source().clone(), write.observation().scope().clone(), @@ -829,9 +866,26 @@ fn canonical_cursor_advance_receipt_remains_typed_after_authority_lookup() { ) .unwrap(); + execute_cursor_advance(&mut connection, &conflicting).unwrap(); + assert_eq!( + connection + .query_row( + "SELECT reason, receipt_id FROM source_cursor_advances", + [], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .unwrap(), + ( + ObservationCoverageReason::DuplicateObservation + .as_str() + .to_owned(), + "receipt.fixture".to_owned(), + ) + ); + restore_source_cursor(&connection, &observation_cursor); let error = execute_cursor_advance(&mut connection, &conflicting).unwrap_err(); let StorageOperationError::CursorAdvanceLedgerDisagreement { disagreement } = error else { - panic!("expected structured immutable ledger disagreement"); + panic!("expected structured immutable ledger disagreement, got {error:?}"); }; assert!(matches!( disagreement.stored().receipt_id(), @@ -850,6 +904,7 @@ fn corrupt_cursor_advance_ledger_values_are_opaque_and_content_free() { let mut connection = connection(); let write = anchored_observation_write("fixture", "receipt.fixture"); execute(&mut connection, &write).unwrap(); + let observation_cursor = source_cursor_json(&connection); let advance = ObservationCursorAdvance::for_ordering( write.observation().source().clone(), write.observation().scope().clone(), @@ -879,10 +934,11 @@ fn corrupt_cursor_advance_ledger_values_are_opaque_and_content_free() { ObservationCoverageReason::OutOfScope, ) .unwrap(); + restore_source_cursor(&connection, &observation_cursor); let error = execute_cursor_advance(&mut connection, &conflicting).unwrap_err(); let StorageOperationError::CursorAdvanceLedgerDisagreement { disagreement } = error else { - panic!("expected structured immutable ledger disagreement"); + panic!("expected structured immutable ledger disagreement, got {error:?}"); }; assert!(matches!( disagreement.stored().reason(), @@ -912,6 +968,7 @@ fn short_corrupt_ledger_receipt_stays_opaque_across_runtime_boundary() { let mut connection = connection(); let write = anchored_observation_write("fixture", "receipt.fixture"); execute(&mut connection, &write).unwrap(); + let observation_cursor = source_cursor_json(&connection); let advance = ObservationCursorAdvance::for_ordering( write.observation().source().clone(), write.observation().scope().clone(), @@ -942,10 +999,11 @@ fn short_corrupt_ledger_receipt_stays_opaque_across_runtime_boundary() { ObservationCoverageReason::OutOfScope, ) .unwrap(); + restore_source_cursor(&connection, &observation_cursor); let error = execute_cursor_advance(&mut connection, &conflicting).unwrap_err(); let StorageOperationError::CursorAdvanceLedgerDisagreement { disagreement } = error else { - panic!("expected structured immutable ledger disagreement"); + panic!("expected structured immutable ledger disagreement, got {error:?}"); }; assert!(matches!( disagreement.stored().receipt_id(), diff --git a/crates/tracedecay-search-eval/src/bin/tracedecay-search-eval-direct.rs b/crates/tracedecay-search-eval/src/bin/tracedecay-search-eval-direct.rs index eca9235503..b353907380 100644 --- a/crates/tracedecay-search-eval/src/bin/tracedecay-search-eval-direct.rs +++ b/crates/tracedecay-search-eval/src/bin/tracedecay-search-eval-direct.rs @@ -239,14 +239,15 @@ mod tests { } #[test] - fn default_validation_uses_the_byte_pinned_packaged_workload() { + fn default_validation_binds_the_packaged_workload_identity() { let summary = validate_requested_workload(std::path::Path::new("."), None) .expect("packaged workload validates"); assert_eq!(summary.status, DirectEvaluationStatusV1::Pass); assert_eq!( summary.workload_digest, - "sha256:8657aa486a4c58e17c9969c7aa5d143a4d30e88dca7d26f13e61c7d3effab091" + tracedecay_query::search_quality::packaged::WORKLOAD_SHA256, + "validate must return the packaged workload identity, not a second pin" ); assert_eq!(summary.profile_count, 1); assert_eq!(summary.query_count, 67); diff --git a/crates/tracedecay-search-eval/src/candidate_output.rs b/crates/tracedecay-search-eval/src/candidate_output.rs index 35167dbfcd..3ce4a90842 100644 --- a/crates/tracedecay-search-eval/src/candidate_output.rs +++ b/crates/tracedecay-search-eval/src/candidate_output.rs @@ -5,7 +5,7 @@ //! lexical, and graph production lanes. //! //! Outputs deterministic checked-in `train` / `validation` candidate records -//! plus current/10x resource samples and fallback digests. Cancellation is +//! plus current/10x resource samples and ranking receipts. Cancellation is //! proved fail-closed before those records are returned; it is not restamped //! as a policy field. Labels are ordinary reviewable fixture data, never a //! production authority. @@ -593,10 +593,32 @@ fn generate_partition_output( let peak_before = peak_rss_bytes(); for query in &queries { let started = Instant::now(); - // The row and both partition fallback digests share one composition. + // The row and both partition receipts share one composition. The + // receipt is the ordered ranking, not the generation-scoped fallback + // subpayload: that digest moves whenever extractor revisions reseal + // the generation every occurrence id names. let composed = compose_production_query(published, profile, query)?; - let fallback = query_fallback_from_composition(&composed)?; - fallback_digests.push((query.query_id.as_str(), fallback.digest.as_str().to_owned())); + let ranked = map_ranked_candidates(published, &composed)?; + let coverage = query_lane_coverage(&composed); + // The subpayload's digest is unfit as a ranking pin; its contract is + // not. Constructing it still proves canonical `final_ordinal` order, + // per-candidate validity, and query-fallback-only contributions for + // every composed query. + QueryFallbackSubpayload::new( + composed.profile_id.clone(), + composed.ranked_candidates.clone(), + coverage.clone(), + composed.freshness.clone(), + None, + ) + .map_err(|error| CandidateOutputError::Contract(error.to_string()))?; + let receipt = ranking_receipt_digest( + composed.profile_id.as_str(), + query.query_id.as_str(), + &coverage, + &ranked, + )?; + fallback_digests.push((query.query_id.as_str(), receipt)); rows.push(query_row_from_composition(published, query, &composed)?); latencies_us.push(started.elapsed().as_micros() as u64); } @@ -609,7 +631,7 @@ fn generate_partition_output( ); let fallback_digest = canonical_sha256(&( - "tracedecay.search-eval.partition-fallbacks.v1", + "tracedecay.search-eval.partition-rankings.v1", &fallback_digests, ))?; let query_digest = fallback_digest.clone(); @@ -839,9 +861,9 @@ fn compose_production_query( .map_err(|error| CandidateOutputError::Contract(error.to_string())) } -fn query_fallback_from_composition( +fn query_lane_coverage( output: &CompositionOutputV1, -) -> Result { +) -> BTreeMap { let mut coverage = BTreeMap::new(); for lane in RetrieverKind::QUERY_FALLBACK_LANES { coverage.insert( @@ -853,41 +875,50 @@ fn query_fallback_from_composition( .unwrap_or(PublicRetrieverStatus::Unavailable), ); } - let fallback = QueryFallbackSubpayload::new( - output.profile_id.clone(), - output.ranked_candidates.clone(), - coverage, - output.freshness.clone(), - None, - ) - .map_err(|error| CandidateOutputError::Contract(error.to_string()))?; - fallback - .validate() - .map_err(|error| CandidateOutputError::Contract(error.to_string()))?; - Ok(fallback) + coverage +} + +/// Ranking identity the search-eval pins compare. +/// +/// Production `QueryFallbackSubpayload` digests stay generation-scoped: every +/// lexical occurrence id is `code-chunk:{generation}:{chunk}`, and the eval +/// request's freshness digest names that generation, whose fingerprint includes +/// extractor revisions. Hashing that subpayload made an extractor revision +/// bump look like a ranking change. This receipt hashes the generation-free +/// rows the quality report already scores, plus the public lane coverage. +fn ranking_receipt_digest( + profile_id: &str, + query_id: &str, + lane_coverage: &BTreeMap, + ranked: &[RankedCandidateRowV1], +) -> Result { + canonical_sha256(&( + "tracedecay.search-eval.ranking-receipt.v1", + profile_id, + query_id, + lane_coverage, + ranked, + )) } fn map_ranked_candidates( published: &PublishedCorpus, output: &CompositionOutputV1, ) -> Result, CandidateOutputError> { - map_ranked_candidate_list(published, &output.ranked_candidates) + map_ranked_candidate_list(&published.occurrence_map, &output.ranked_candidates) } fn map_ranked_candidate_list( - published: &PublishedCorpus, + occurrence_map: &BTreeMap, ranked_candidates: &[tracedecay_domain::RankedCandidate], ) -> Result, CandidateOutputError> { let mut rows = Vec::new(); for ranked in ranked_candidates { - let entry = published - .occurrence_map + let entry = occurrence_map .get(ranked.candidate.anchor_id.as_str()) .or_else(|| { ranked.candidate.occurrences.iter().find_map(|occurrence| { - published - .occurrence_map - .get(occurrence.source_occurrence_id.as_str()) + occurrence_map.get(occurrence.source_occurrence_id.as_str()) }) }) .cloned() @@ -1541,6 +1572,129 @@ pub(crate) mod tests { packaged_fixture().workload().clone() } + fn ranked_with_generation(generation: &str) -> tracedecay_domain::RankedCandidate { + let occurrence = tracedecay_domain::OccurrenceProvenance { + source_occurrence_id: id(&format!("code-chunk:{generation}:chunk.stable")) + .expect("occurrence id"), + file_occurrence_id: Some(id("file.time").expect("file id")), + retriever_evidence_anchor: tracedecay_domain::RetrievalAnchorId::new("evidence.stable") + .expect("evidence anchor"), + source_namespace: id("ns.code.daemon").expect("namespace"), + repository_id: None, + session_or_thread_id: None, + logical_copy_cluster_id: None, + logical_copy_evidence_anchor: None, + evidence_role: tracedecay_domain::EvidenceRole::Primary, + freshness: tracedecay_domain::SourceFreshness { + source_namespace: id("ns.code.daemon").expect("freshness namespace"), + source_instance: id("instance.code-index.daemon").expect("instance"), + source_watermark: Some(1), + projection_watermark: Some(1), + observed_at: UtcMicros(1_000_000), + source_generation: Some(1), + generation_lag: Some(0), + compatibility: tracedecay_domain::FreshnessCompatibilityV1::Current, + policy_revision: id("policy.candidate.v1").expect("policy"), + }, + }; + tracedecay_domain::RankedCandidate { + candidate: tracedecay_domain::FusedCandidate { + anchor_id: tracedecay_domain::RetrievalAnchorId::new("code-symbol:symbol.stable") + .expect("anchor"), + logical_evidence_id: id("code-symbol:symbol.stable").expect("evidence"), + occurrences: vec![occurrence], + exact_class: ExactClass::Approximate, + utility_micros: 1, + contributions: Vec::new(), + freshness: Vec::new(), + decisions: Vec::new(), + }, + final_ordinal: 0, + } + } + + /// Extractor revision bumps reseal the generation embedded in every + /// `code-chunk:{generation}:{chunk}` occurrence id. The production + /// fallback digest binds that id; the ranking receipt must not. + #[test] + fn ranking_receipt_ignores_generation_scoped_occurrence_ids() { + let generation_a = "generation.v1.aaaaaaaa.00000001"; + let generation_b = "generation.v1.bbbbbbbb.00000002"; + let mut map = BTreeMap::new(); + map.insert( + "code-symbol:symbol.stable".to_owned(), + OccurrenceMapEntry { + document_id: "time".to_owned(), + scope: "research".to_owned(), + display_anchors: vec!["time::UtcMicros".to_owned()], + }, + ); + let rows_a = map_ranked_candidate_list(&map, &[ranked_with_generation(generation_a)]) + .expect("map generation a"); + let rows_b = map_ranked_candidate_list(&map, &[ranked_with_generation(generation_b)]) + .expect("map generation b"); + assert_eq!( + rows_a, rows_b, + "display rows are keyed by the generation-free anchor" + ); + + let coverage = BTreeMap::from([ + (RetrieverKind::ExactLiteral, PublicRetrieverStatus::Complete), + (RetrieverKind::Lexical, PublicRetrieverStatus::Complete), + (RetrieverKind::Graph, PublicRetrieverStatus::Unavailable), + ]); + let receipt = + |rows: &[RankedCandidateRowV1], + lanes: &BTreeMap| { + ranking_receipt_digest("profile.query-fallback", "train-001", lanes, rows) + .expect("ranking receipt") + }; + let receipt_a = receipt(&rows_a, &coverage); + assert_eq!(receipt_a, receipt(&rows_b, &coverage)); + + let mut coverage_changed = coverage.clone(); + coverage_changed.insert(RetrieverKind::Graph, PublicRetrieverStatus::Complete); + assert_ne!( + receipt_a, + receipt(&rows_a, &coverage_changed), + "lane coverage is part of the ranking receipt" + ); + let mut reordered = rows_a; + reordered.push(RankedCandidateRowV1 { + anchor: "code-chunk:chunk.other".to_owned(), + anchors: vec!["watermark::merge_max".to_owned()], + scope: "research".to_owned(), + document_id: "watermark".to_owned(), + tier: "approximate".to_owned(), + }); + assert_ne!( + receipt_a, + receipt(&reordered, &coverage), + "a different ranked set must move the receipt" + ); + + let production_digest = |generation: &str| { + let lanes = RetrieverKind::QUERY_FALLBACK_LANES + .into_iter() + .map(|lane| (lane, PublicRetrieverStatus::Complete)) + .collect(); + tracedecay_domain::QueryFallbackSubpayload::new( + id("profile.query-fallback").expect("profile"), + vec![ranked_with_generation(generation)], + lanes, + Vec::new(), + None, + ) + .expect("production fallback subpayload") + .digest + }; + assert_ne!( + production_digest(generation_a).as_str(), + production_digest(generation_b).as_str(), + "the production fallback digest still moves with the sealed generation" + ); + } + #[test] fn fusion_profile_carries_the_checked_in_lane_weights() { let workload = workload(); diff --git a/crates/tracedecay-search-eval/src/lib.rs b/crates/tracedecay-search-eval/src/lib.rs index 27e91ed35c..41091a12fd 100644 --- a/crates/tracedecay-search-eval/src/lib.rs +++ b/crates/tracedecay-search-eval/src/lib.rs @@ -54,10 +54,11 @@ pub fn default_workload_path(repo_root: &Path) -> PathBuf { repo_root.join(WORKLOAD_RELATIVE) } -/// Validate the byte-pinned packaged workload. +/// Validate the packaged workload identity. /// /// Ordinary developer comparisons may use an explicit workload; this default -/// fixture is the one whose digest the package pins. +/// fixture is the one whose input digest the package pins. Ranking receipts +/// are checked separately and are not part of that identity. pub fn validate_default_workload() -> Result { let assets = packaged_assets::materialize()?; validate_direct_workload(assets.root(), Some(&assets.workload_path())) diff --git a/crates/tracedecay-search-eval/src/report_tests.rs b/crates/tracedecay-search-eval/src/report_tests.rs index 9d1aaa508f..fa46c7fdfc 100644 --- a/crates/tracedecay-search-eval/src/report_tests.rs +++ b/crates/tracedecay-search-eval/src/report_tests.rs @@ -38,8 +38,9 @@ fn baseline_report_retains_raw_fallback_current_and_exact_ten_x_samples() { .expect("generate direct fixture outputs"); let report = evaluate_generated_outputs(repo_root, workload, &generated) .expect("evaluate direct fixture outputs"); - // Production retrieval changes must land with a re-pinned workload; the - // pin is what turns a silent ranking change into a visible one. + // A ranking change must move the receipt. A generation reseal must not: + // the receipt hashes ordered rows and lane coverage, not the sealed + // generation those rows were bound under. for profile in &report.profiles { let observed = generated .outputs @@ -56,11 +57,12 @@ fn baseline_report_retains_raw_fallback_current_and_exact_ten_x_samples() { .unwrap_or_else(|| "no generated output for this profile".to_owned()); assert!( profile.fallback_matches_expected, - "{}:{} query fallback digest drifted from \ + "{}:{} ranking receipt drifted from \ `expected_query_fallback_digests.{}` in \ - tests/fixtures/search_quality/query-lexical-graph-workload-v1.json \ - ({observed}). Confirm the new query results are intended, then re-pin \ - the packaged workload, packaged::WORKLOAD_SHA256, and the workload digest pins.", + crates/tracedecay-query/assets/runtime-root/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json \ + ({observed}). The receipt binds ordered ranking rows and lane coverage, \ + not generation or extractor-revision identity. Re-pin only that receipt \ + when the ranking itself changed; do not touch the workload identity pin.", profile.profile_id, profile.partition, profile.partition ); } diff --git a/crates/tracedecay-session-temporal-store/src/registered_lcm_render.rs b/crates/tracedecay-session-temporal-store/src/registered_lcm_render.rs index 81c24492ca..a1c09679a3 100644 --- a/crates/tracedecay-session-temporal-store/src/registered_lcm_render.rs +++ b/crates/tracedecay-session-temporal-store/src/registered_lcm_render.rs @@ -304,40 +304,64 @@ async fn raw_message_overviews( provider: &str, session_id: &str, ) -> Result, LcmError> { + // The snippet is the bounded preview. `total_chars` is the message's own + // length: an external payload's recorded char count, otherwise the stored + // content. Using the snippet length here described a stub, which is how a + // session that expand can read came back empty. + let preview_cap = i64::try_from(tracedecay_lcm::MAX_DERIVED_SNIPPET_CHARS) + .map_err(|_| LcmError::Db("snippet preview cap does not fit i64".to_string()))?; let mut rows = query( snapshot, - "SELECT message_id, store_id, role, storage_kind, payload_ref, - LENGTH(snippet_text) - FROM lcm_raw_messages - WHERE provider = ?1 AND session_id = ?2 - ORDER BY store_id + "SELECT raw.message_id, raw.store_id, raw.role, raw.storage_kind, raw.payload_ref, + CASE + WHEN raw.snippet_text <> '' THEN raw.snippet_text + ELSE substr(COALESCE(raw.content, ''), 1, ?3) + END, + COALESCE( + (SELECT payload.char_count + FROM lcm_external_payloads AS payload + WHERE payload.payload_ref = raw.payload_ref), + length(raw.content), + length(raw.snippet_text), + 0 + ) + FROM lcm_raw_messages AS raw + WHERE raw.provider = ?1 AND raw.session_id = ?2 + ORDER BY raw.store_id LIMIT 20", - params![provider, session_id], + params![provider, session_id, preview_cap], ) .await?; let mut out = Vec::new(); while let Some(row) = next_row(&mut rows).await? { let storage_kind_text: String = field!(&row, 3)?; - let total_chars = field!(&row, 5, i64)?.max(0) as u64; + let content_preview: String = field!(&row, 5)?; + let total_chars = field!(&row, 6, i64)?.max(0) as u64; out.push(LcmRawMessageOverview { message_id: field!(&row, 0)?, store_id: field!(&row, 1)?, role: field!(&row, 2)?, storage_kind: storage_kind(&storage_kind_text)?, payload_ref: field!(&row, 4)?, - content_preview: String::new(), - content_range: LcmContentRange { - offset: 0, - limit: 0, - returned_chars: 0, - total_chars, - truncated: total_chars > 0, - }, + content_range: preview_range(&content_preview, total_chars), + content_preview, }); } Ok(out) } +fn preview_range(preview: &str, total_chars: u64) -> LcmContentRange { + let returned_chars = preview.chars().count() as u64; + let total_chars = total_chars.max(returned_chars); + LcmContentRange { + offset: 0, + limit: returned_chars, + returned_chars, + total_chars, + truncated: returned_chars < total_chars, + } +} + async fn summary_overviews( snapshot: &(impl QueryExecutor + ?Sized), provider: &str, @@ -346,7 +370,7 @@ async fn summary_overviews( ) -> Result, LcmError> { let mut rows = query( snapshot, - "SELECT node_id, conversation_id, depth, created_at + "SELECT node_id, conversation_id, depth, summary_text, created_at FROM lcm_summary_nodes WHERE provider = ?1 AND session_id = ?2 ORDER BY depth, created_at, node_id @@ -357,14 +381,17 @@ async fn summary_overviews( let mut out = Vec::new(); while let Some(row) = next_row(&mut rows).await? { let node_id: String = field!(&row, 0)?; + let summary_text: String = field!(&row, 3)?; let source_count = relation(relations, &node_id)?.sources.len(); out.push(LcmSummaryNodeOverview { node_id, conversation_id: field!(&row, 1)?, depth: field!(&row, 2)?, - summary_preview: String::new(), + summary_preview: tracedecay_lcm::retrieval_content::derived_text_for_snippet( + &summary_text, + ), source_count, - created_at: field!(&row, 3)?, + created_at: field!(&row, 4)?, }); } Ok(out) @@ -489,6 +516,14 @@ async fn describe_external_payload( if payload.provider != provider || payload.session_id != session_id { return Err(LcmError::PayloadNotFound); } + let content_preview = external_payload_preview( + snapshot, + provider, + session_id, + &payload.message_id, + payload_ref, + ) + .await?; Ok(LcmDescribeExternalPayload { payload_ref: payload.payload_ref, provider: payload.provider, @@ -500,10 +535,35 @@ async fn describe_external_payload( char_count: payload.char_count, created_at: payload.created_at, metadata_json: payload.metadata_json, - content_preview: String::new(), + content_preview, }) } +async fn external_payload_preview( + snapshot: &(impl QueryExecutor + ?Sized), + provider: &str, + session_id: &str, + message_id: &str, + payload_ref: &str, +) -> Result { + let mut rows = query( + snapshot, + "SELECT snippet_text + FROM lcm_raw_messages + WHERE provider = ?1 + AND session_id = ?2 + AND message_id = ?3 + AND payload_ref = ?4 + LIMIT 1", + params![provider, session_id, message_id, payload_ref], + ) + .await?; + if let Some(row) = next_row(&mut rows).await? { + return field!(&row, 0); + } + Ok(format!("[externalized payload ref={payload_ref}]")) +} + /// Loads the raw row a directly requested `store_id` names, refusing when it is /// gone. Summary *lineage* reads must use [`find_raw_message`] instead: an /// absent row there is retention, not a missing target. diff --git a/crates/tracedecay-session-temporal-store/src/registered_lcm_render/tests.rs b/crates/tracedecay-session-temporal-store/src/registered_lcm_render/tests.rs index 426598ee66..2dfcd7faec 100644 --- a/crates/tracedecay-session-temporal-store/src/registered_lcm_render/tests.rs +++ b/crates/tracedecay-session-temporal-store/src/registered_lcm_render/tests.rs @@ -393,3 +393,80 @@ async fn registered_metadata_rows_do_not_fabricate_full_raw_messages() { "expected a payload-integrity refusal, got: {error:?}" ); } + +#[tokio::test] +async fn session_describe_reports_the_message_not_an_empty_stub() { + let directory = tempdir().expect("temporary session store"); + let runtime = seeded_render_fixture(directory.path()).await; + let content = "canonical raw message plus hidden tail"; + runtime + .registered_database(HostAdmissionScope::Profile) + .expect("registered session database") + .writer_connection() + .expect("registered writer") + .execute_batch(&format!( + "UPDATE lcm_raw_messages + SET content = '{content}', snippet_text = 'canonical raw' + WHERE message_id = 'message-a';" + )) + .await + .expect("shorten the stored preview without shortening the message"); + let snapshot = runtime + .registered_database(HostAdmissionScope::Profile) + .expect("registered session database") + .read_snapshot() + .await + .expect("registered read snapshot"); + + let description = describe( + &snapshot, + LcmDescribeRequest { + provider: "codex".to_string(), + session_id: "session-a".to_string(), + target: LcmDescribeTarget::Session, + }, + &canonical_fixture_relations(), + ) + .await + .expect("session describe"); + + let overview = description + .raw_messages + .iter() + .find(|message| message.message_id == "message-a") + .expect("describe must list the captured message"); + assert_eq!(overview.content_preview, "canonical raw"); + assert!(!overview.content_preview.contains("hidden tail")); + assert_eq!( + overview.content_range.total_chars, + content.chars().count() as u64 + ); + assert!(overview.content_range.truncated); + let summary = description + .summary_nodes + .iter() + .find(|node| node.node_id == "summary-child") + .expect("describe must list the summary"); + assert_eq!(summary.summary_preview, "canonical child summary"); + + let payload = describe( + &snapshot, + LcmDescribeRequest { + provider: "codex".to_string(), + session_id: "session-a".to_string(), + target: LcmDescribeTarget::ExternalPayload { + payload_ref: "payload-a".to_string(), + }, + }, + &[], + ) + .await + .expect("external payload describe"); + assert_eq!( + payload + .external_payload + .expect("payload metadata") + .content_preview, + "canonical external payload" + ); +} diff --git a/crates/tracedecay-sessions/src/runtime/hosts/cursor.rs b/crates/tracedecay-sessions/src/runtime/hosts/cursor.rs index 4bc895f9b2..16a37a8027 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/cursor.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/cursor.rs @@ -218,6 +218,27 @@ impl CursorSourceAdmissionTally { } } +/// Fold one hook pass's admission over the projection drain it happened to run. +/// +/// The projection queue is per scope. The project catch-up drains it too, so +/// `drain.source_deferred`, `drain.exact_duplicate`, and `drain.messages_upserted` +/// describe whoever last touched that queue, not this pass. A residual there +/// must not hide a commit, invent a duplicate, or turn a byte-finished pass +/// into backpressure. Admission is the commit. +fn account_hook_admission( + mut drain: projection::CursorTranscriptIngestStats, + observations_committed: u64, + fully_replayed: bool, + admission_deferred: bool, + bytes_consumed: u64, +) -> projection::CursorTranscriptIngestStats { + drain.bytes_consumed = bytes_consumed; + drain.source_deferred = admission_deferred; + drain.observations_committed = observations_committed; + drain.exact_duplicate = observations_committed == 0 && fully_replayed; + drain +} + // Cursor JSONL admission chokepoint: the whole per-file admission future is // boxed here so the per-file sweep loop no longer pins each call, keeping the // debug poll frame bounded through the deep ingest recursion chain. @@ -631,19 +652,19 @@ pub async fn try_ingest_cursor_transcript_event_capped_with_admission( admitted.record(&progress); budget.record_progress(progress.bytes_consumed, progress.source_deferred); } - let mut stats = drain_cursor_observation_projections( + let drain = drain_cursor_observation_projections( admission, &scope, &ObservationCancellation::default(), ) .await?; - stats.bytes_consumed = budget.consumed(); - stats.source_deferred |= budget.deferred(); - stats.observations_committed = admitted.observations_committed; - stats.exact_duplicate |= stats.messages_upserted == 0 - && stats.observations_committed == 0 - && admitted.fully_replayed(); - Ok(stats) + Ok(account_hook_admission( + drain, + admitted.observations_committed, + admitted.fully_replayed(), + budget.deferred(), + budget.consumed(), + )) } pub async fn ingest_cursor_user_transcript_event_capped( @@ -781,19 +802,19 @@ pub async fn try_ingest_cursor_user_transcript_event_capped_with_admission( admitted.record(&progress); budget.record_progress(progress.bytes_consumed, progress.source_deferred); } - let mut stats = drain_cursor_observation_projections( + let drain = drain_cursor_observation_projections( admission, &scope, &ObservationCancellation::default(), ) .await?; - stats.bytes_consumed = budget.consumed(); - stats.source_deferred |= budget.deferred(); - stats.observations_committed = admitted.observations_committed; - stats.exact_duplicate |= stats.messages_upserted == 0 - && stats.observations_committed == 0 - && admitted.fully_replayed(); - Ok(stats) + Ok(account_hook_admission( + drain, + admitted.observations_committed, + admitted.fully_replayed(), + budget.deferred(), + budget.consumed(), + )) } pub(in crate::runtime) fn try_ingest_cursor_project_sweep_capped_with_session_ids< diff --git a/crates/tracedecay-sessions/src/runtime/hosts/cursor/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/cursor/tests.rs index b548c3bc95..e80482175c 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/cursor/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/cursor/tests.rs @@ -509,6 +509,52 @@ async fn replayed_cursor_ingest_reports_an_exact_duplicate_not_a_bare_replay() { ); } +/// The shared projection queue's residual is not this pass. A deferred drain +/// with leftover rows does not defer the pass, invent a duplicate, or hide +/// the frames admission persisted. +#[test] +fn hook_admission_ignores_a_shared_drain_residual() { + let committed = account_hook_admission( + CursorTranscriptIngestStats { + messages_upserted: 9, + source_deferred: true, + exact_duplicate: true, + ..CursorTranscriptIngestStats::default() + }, + 2, + false, + false, + 40, + ); + assert_eq!(committed.observations_committed, 2); + assert_eq!(committed.bytes_consumed, 40); + assert_eq!(committed.messages_upserted, 9); + assert!(!committed.source_deferred); + assert!(!committed.exact_duplicate); + + let replayed = account_hook_admission( + CursorTranscriptIngestStats { + messages_upserted: 4, + source_deferred: true, + exact_duplicate: false, + ..CursorTranscriptIngestStats::default() + }, + 0, + true, + false, + 0, + ); + assert_eq!(replayed.observations_committed, 0); + assert!(replayed.exact_duplicate); + assert!(!replayed.source_deferred); + + let deferred = + account_hook_admission(CursorTranscriptIngestStats::default(), 0, false, true, 8); + assert!(deferred.source_deferred); + assert!(!deferred.exact_duplicate); + assert_eq!(deferred.observations_committed, 0); +} + /// The duplicate verdict is evidence, not a default: a source this pass has /// never opened carries no proof that anything was committed before. #[tokio::test] diff --git a/crates/tracedecay-store/src/canonical_projection.rs b/crates/tracedecay-store/src/canonical_projection.rs index 6cb7ed3280..0ee649de97 100644 --- a/crates/tracedecay-store/src/canonical_projection.rs +++ b/crates/tracedecay-store/src/canonical_projection.rs @@ -10,16 +10,75 @@ use tracedecay_domain::{ use crate::cursor_dispatch::{cursor_dispatch_model, dispatch_text, is_subagent_dispatch_tool}; use crate::provider_descriptor::{ - provider_message_semantics, synthesizes_native_record_id, tool_metadata_normalizer, + ProviderMessageSemantics, provider_message_semantics, synthesizes_native_record_id, + tool_metadata_normalizer, }; use crate::{ ObservationProjection, ProjectionSkipReason, ProjectionStoreError, ProjectionStoreResult, SessionMessageRecord, SessionRecord, WorkflowFactRecord, }; -#[hotpath::measure(label = "store.projection.derive_canonical")] +/// Which projector rendering to derive. +/// +/// Releases through v0.1.0-beta.37 wrote `ShippedRelease`. The reducer is +/// otherwise unchanged; only Codex goal-context semantics were added after +/// that tag. +#[derive(Clone, Copy, Eq, PartialEq)] +enum CanonicalRendering { + Current, + ShippedRelease, +} + +/// Codex goal-context semantics are the only post-release rendering. A shipped +/// derivation withholds them. Every other field is this reducer. +fn rendering_message_semantics( + rendering: CanonicalRendering, + provider: &str, + native_record_kind: &str, + role: &str, + content: &serde_json::Value, + has_native_item_identity: bool, +) -> Option { + match rendering { + CanonicalRendering::ShippedRelease => None, + CanonicalRendering::Current => provider_message_semantics( + provider, + native_record_kind, + role, + content, + has_native_item_identity, + ), + } +} + pub fn derive_canonical_projection( observation: &DurableObservationV1, +) -> ProjectionStoreResult { + derive_canonical_projection_for(observation, CanonicalRendering::Current) +} + +/// Whether `stored` is the message row a shipped release wrote for `observation`. +/// +/// A current-provenance row that still holds that rendering is an interrupted +/// write. Any other body, including a derivation that does not complete, is not. +pub fn stored_message_is_shipped_release_rendering( + observation: &DurableObservationV1, + stored: &SessionMessageRecord, +) -> bool { + let Ok(released) = + derive_canonical_projection_for(observation, CanonicalRendering::ShippedRelease) + else { + return false; + }; + released + .messages() + .any(|projection| projection.message() == stored) +} + +#[hotpath::measure(label = "store.projection.derive_canonical")] +fn derive_canonical_projection_for( + observation: &DurableObservationV1, + rendering: CanonicalRendering, ) -> ProjectionStoreResult { let envelope = CanonicalObservationEnvelopeV1::deserialize(observation.payload()).map_err(|_| { @@ -42,7 +101,7 @@ pub fn derive_canonical_projection( )); } - let mut projected = canonical_message_fields(&envelope)?; + let mut projected = canonical_message_fields_for(rendering, &envelope)?; let session_fields = if envelope.provider().as_str() == "claude" { None } else { @@ -115,7 +174,8 @@ pub fn derive_canonical_projection( let ordinal = i64::try_from(ordinal).map_err(|_| { ProjectionStoreError::Contract(ObservationContractError::InvalidCanonicalPayload) })?; - let metadata_json = canonical_message_metadata( + let metadata_json = canonical_message_metadata_for( + rendering, &envelope, (!session_metadata.is_empty()).then_some(&session_metadata), )?; @@ -310,9 +370,18 @@ fn canonical_session_metadata( serialize_metadata_map(&canonical_session_metadata_map(provider, session)) } +#[cfg(test)] fn canonical_message_metadata( envelope: &CanonicalObservationEnvelopeV1, session_metadata: Option<&serde_json::Map>, +) -> ProjectionStoreResult { + canonical_message_metadata_for(CanonicalRendering::Current, envelope, session_metadata) +} + +fn canonical_message_metadata_for( + rendering: CanonicalRendering, + envelope: &CanonicalObservationEnvelopeV1, + session_metadata: Option<&serde_json::Map>, ) -> ProjectionStoreResult { let serde_json::Value::Object(mut metadata) = serde_json::to_value(envelope) .map_err(|_| ProjectionStoreError::Contract(ObservationContractError::CanonicalEncoding))? @@ -333,7 +402,8 @@ fn canonical_message_metadata( .facts() .iter() .find(|fact| matches!(fact, CanonicalObservationFactV1::Message { .. })) - && let Some(semantics) = provider_message_semantics( + && let Some(semantics) = rendering_message_semantics( + rendering, envelope.provider().as_str(), envelope.native_record_kind(), canonical_role(*role), @@ -687,8 +757,16 @@ fn canonical_cursor_compatibility_message_fields( Ok((primary_message_id, derived)) } +#[cfg(test)] fn canonical_message_fields( envelope: &CanonicalObservationEnvelopeV1, +) -> ProjectionStoreResult> { + canonical_message_fields_for(CanonicalRendering::Current, envelope) +} + +fn canonical_message_fields_for( + rendering: CanonicalRendering, + envelope: &CanonicalObservationEnvelopeV1, ) -> ProjectionStoreResult> { let facts = envelope.facts(); let tool_names = facts @@ -711,7 +789,8 @@ fn canonical_message_fields( { let role = canonical_role(*role); let text = canonical_fact_text(content)?; - if let Some(semantics) = provider_message_semantics( + if let Some(semantics) = rendering_message_semantics( + rendering, envelope.provider().as_str(), envelope.native_record_kind(), role, diff --git a/crates/tracedecay-store/src/lib.rs b/crates/tracedecay-store/src/lib.rs index c09436af3b..7d85f4f7c1 100644 --- a/crates/tracedecay-store/src/lib.rs +++ b/crates/tracedecay-store/src/lib.rs @@ -33,7 +33,8 @@ pub mod session; pub mod transcript; pub use canonical_projection::{ - canonical_fact_text, derive_canonical_projection, workflow_semantic_kind, + canonical_fact_text, derive_canonical_projection, stored_message_is_shipped_release_rendering, + workflow_semantic_kind, }; pub use codex_goal_context::{ CodexGoalContext, CodexGoalContextCorrelation, CodexGoalContextSource, diff --git a/crates/tracedecay/Cargo.toml b/crates/tracedecay/Cargo.toml index 2a7f1cf535..a6616a03e4 100644 --- a/crates/tracedecay/Cargo.toml +++ b/crates/tracedecay/Cargo.toml @@ -231,6 +231,14 @@ test-transport = [ "tracedecay-code-index-runtime/test-transport", ] +# The evaluator library is the only selector of `tracedecay-query/search-eval`, +# the eval-only in-memory lexical projection. Cargo rejects optional +# dev-dependencies, and an unconditional one unifies this feature into every +# test target, so the transport suites compile that projection. Journeys that +# compare the CLI receipt to the library enable this feature. `test-transport` +# and `production` must not. +search-eval = ["dep:tracedecay-search-eval"] + # The typed RMCP benchmark is the only consumer of the client-side RMCP # transport. Keep that surface out of ordinary integration-fixture builds: # the benchmark enables its complete client/runtime dependency set explicitly. @@ -340,6 +348,10 @@ keyring = { version = "4.1.5", features = ["v1"] } tempfile = "3" futures-util = "0.3.33" rmcp = { version = "3.0.1", default-features = false, features = ["server"] } +# Opt-in. Cargo rejects optional dev-dependencies, and an unconditional one +# unifies `tracedecay-query/search-eval` into every test target. Default and +# `production` builds leave it off, so the shipped CLI does not link it. +tracedecay-search-eval = { path = "../tracedecay-search-eval", version = "0.1.0", optional = true } # `kill(2)` for the daemon integration suites' physical-restart journeys # (tests/daemon_suite). Store-locality detection moved with the locator @@ -348,7 +360,6 @@ rmcp = { version = "3.0.1", default-features = false, features = ["server"] } libc = "0.2" [dev-dependencies] -tracedecay-search-eval = { path = "../tracedecay-search-eval", version = "0.1.0" } tree-sitter = "0.26" rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } diff --git a/crates/tracedecay/src/daemon/core_client.rs b/crates/tracedecay/src/daemon/core_client.rs index a81110e2b2..0d7cebc839 100644 --- a/crates/tracedecay/src/daemon/core_client.rs +++ b/crates/tracedecay/src/daemon/core_client.rs @@ -26,11 +26,6 @@ use super::{ TraceDecayError, error_is_project_open_retryable, tool_call_transport_error_is_retryable, }; -/// Completed retryable problem results to observe before returning the typed -/// state to an interactive caller. Transport-level project-open errors are not -/// results and continue to use their explicit deadline. -const MAX_COMPLETED_TOOL_RESULT_ATTEMPTS: usize = 3; - /// Bounded grace a client keeps reading for *after* the caller's request /// deadline has elapsed. /// @@ -502,32 +497,29 @@ fn daemon_tool_call_error(error: JsonRpcError) -> TraceDecayError { } } -/// The delay a completed tool result directs before the same request is sent -/// again, when its typed problem is a retryable pre-admission state. +/// The delay before re-sending a completed tool result, when that result is +/// the publication-window mounting refusal. /// -/// A project-scoped owner that registers behind the core publication (the -/// retained memory authority, the configuration runtime) answers a -/// `RetryDirective::AfterDelay` unavailable while it is still mounting. The -/// daemon renders that record under the tool result's `problem` member, so -/// the one-shot client reads the directive from the same field every MCP -/// client does. An admitted terminal (a partial effect, a permanent owner -/// failure) never directs a delay and is the answer. +/// A project-scoped owner that registers behind the core publication answers +/// `application.runtime.mounting` while it is still mounting. The daemon +/// renders that record under the tool result's `problem` member. An admitted +/// terminal, and every other completed problem (a retained authority that is +/// unavailable, a saturated owner, an observed diagnostic), is the answer: +/// its `after_delay` directive is for the caller, not a transport loop. fn tool_result_retry_after_delay(result: &serde_json::Value) -> Option { let record: tracedecay_contracts::ApplicationProblemRecord = serde_json::from_value(result.get("problem")?.clone()).ok()?; - record.pre_admission_retry_delay() + record.owner_mount_resend_delay() } /// How long to wait before re-sending the request whose outcome is `result`, /// or `None` when that outcome is the answer. /// -/// Two states are ridden out: the daemon's project-open refusal (a JSON-RPC -/// error carrying the warming hint or a saturated open queue) on the client's -/// own cadence, and a completed result whose typed problem directs an -/// after-delay retry, on the delay the directive names. Project-open errors -/// may wait to `deadline`; completed results are also capped by -/// [`MAX_COMPLETED_TOOL_RESULT_ATTEMPTS`] so a persistent authority result is -/// returned instead of hidden behind a reconnect loop. +/// Two states are ridden out to `deadline`: the daemon's project-open refusal +/// (a JSON-RPC error carrying the warming hint or a saturated open queue) on +/// the client's own cadence, and a completed mounting refusal on the delay +/// that result names. Every other completed result is returned on the first +/// observation. fn project_open_retry_wait( result: &Result, deadline: Instant, @@ -554,7 +546,6 @@ async fn call_tool_with_project_open_retry( tool_name: &str, arguments: serde_json::Value, deadline: Instant, - mut completed_result_attempts: usize, ) -> Result { loop { let result = call_tool_within( @@ -565,12 +556,6 @@ async fn call_tool_with_project_open_retry( deadline, ) .await; - if result.is_ok() { - completed_result_attempts = completed_result_attempts.saturating_add(1); - if completed_result_attempts >= MAX_COMPLETED_TOOL_RESULT_ATTEMPTS { - return result; - } - } let Some(wait) = project_open_retry_wait(&result, deadline) else { return result; }; @@ -584,8 +569,9 @@ async fn call_tool_with_project_open_retry( /// accepting daemon. The request deadline travels on the wire; the local read /// waits that deadline plus the 30s response grace. A warming project, or an /// owner still mounting behind its core publication, still retries for at -/// most the 15s open grace, never past this envelope. Callers that need a -/// different budget use [`call_default_tool_within`] or +/// most the 15s open grace, never past this envelope. A completed result that +/// is not that mounting refusal is returned on the first observation. +/// Callers that need a different budget use [`call_default_tool_within`] or /// [`call_default_tool_awaiting_project_open`]. pub async fn call_default_tool( handshake: &DaemonHandshake, @@ -613,7 +599,6 @@ pub async fn call_default_tool( tool_name, arguments, retry_deadline, - usize::from(result.is_ok()), ) .await } @@ -637,8 +622,8 @@ pub async fn call_default_tool_within( /// Bootstrap callers deliberately trigger the cold open they are waiting for, /// so a transport-level warming hint is progress rather than an answer: /// `tracedecay init` asks for a status it can only get after the open completes. -/// Completed application problems are different: after three identical -/// results, `tracedecay tool` returns that typed state for the caller to decide. +/// A completed mounting refusal is the same kind of progress and is re-sent +/// until `deadline`. Every other completed result is returned immediately. pub async fn call_default_tool_awaiting_project_open( handshake: &DaemonHandshake, tool_name: &str, @@ -646,8 +631,7 @@ pub async fn call_default_tool_awaiting_project_open( deadline: Instant, ) -> Result { let socket_path = default_available_socket_path()?; - call_tool_with_project_open_retry(&socket_path, handshake, tool_name, arguments, deadline, 0) - .await + call_tool_with_project_open_retry(&socket_path, handshake, tool_name, arguments, deadline).await } /// Extracts the single JSON payload from an MCP tool result while ignoring diff --git a/crates/tracedecay/src/daemon/invocation_dispatch.rs b/crates/tracedecay/src/daemon/invocation_dispatch.rs index 61770b6a0f..517134a51c 100644 --- a/crates/tracedecay/src/daemon/invocation_dispatch.rs +++ b/crates/tracedecay/src/daemon/invocation_dispatch.rs @@ -118,9 +118,11 @@ fn lsp_project_open_wait_response( ) -> Option { match outcome { ProjectOpenWaitOutcome::Completed | ProjectOpenWaitOutcome::NotTracked => None, - ProjectOpenWaitOutcome::Failed(error) => Some(DaemonInvocationResponse::problem( + ProjectOpenWaitOutcome::Failed(error) => Some(project_open_refusal_response( request_id.to_owned(), - project_open_problem(&error, workflow_application, git_operation), + &error, + workflow_application, + git_operation, )), ProjectOpenWaitOutcome::Cancelled => Some(DaemonInvocationResponse::application_problem( request_id.to_owned(), @@ -228,9 +230,11 @@ async fn open_scope_set_cas_projects<'a>( Ok(Ok(_)) => {} Ok(Err(error)) => { record_project_open_refusal("multi_root_scope_set_compare_and_swap", &error); - return Err(DaemonInvocationResponse::problem( + return Err(project_open_refusal_response( request_id.to_owned(), - project_open_problem(&error, false, false), + &error, + false, + false, )); } Err(problem) => { @@ -272,9 +276,11 @@ async fn open_scope_set_cas_projects<'a>( Ok(Ok(project_server)) => servers.push(project_server), Ok(Err(error)) => { record_project_open_refusal("multi_root_scope_set_compare_and_swap", &error); - return Err(DaemonInvocationResponse::problem( + return Err(project_open_refusal_response( request_id.to_owned(), - project_open_problem(&error, false, false), + &error, + false, + false, )); } Err(problem) => { @@ -362,9 +368,11 @@ pub(super) async fn execute_portable_daemon_invocation( ); if let Err(error) = project_server { record_project_open_refusal(request.operation().as_str(), &error); - return DaemonInvocationResponse::problem( + return project_open_refusal_response( request_id, - project_open_problem(&error, workflow_application, git_operation), + &error, + workflow_application, + git_operation, ); } let project_route = project_route_for_handshake(handshake); @@ -423,9 +431,11 @@ pub(super) async fn execute_portable_daemon_invocation( } }; if let Err(error) = project_server { - return DaemonInvocationResponse::problem( + return project_open_refusal_response( request_id, - project_open_problem(&error, workflow_application, git_operation), + &error, + workflow_application, + git_operation, ); } let Ok((canonical_project_path, _)) = project_route_for_handshake(handshake) else { @@ -728,9 +738,11 @@ pub(super) async fn execute_daemon_invocation( ); if let Err(error) = project_server { record_project_open_refusal(request.operation().as_str(), &error); - return DaemonInvocationResponse::problem( + return project_open_refusal_response( request_id, - project_open_problem(&error, workflow_application, git_operation), + &error, + workflow_application, + git_operation, ); } let project_route = DaemonEngine::project_route(handshake); @@ -779,9 +791,11 @@ pub(super) async fn execute_daemon_invocation( } }; if let Err(error) = project_server { - return DaemonInvocationResponse::problem( + return project_open_refusal_response( request_id, - project_open_problem(&error, workflow_application, git_operation), + &error, + workflow_application, + git_operation, ); } let Ok((canonical_project_path, _)) = DaemonEngine::project_route(handshake) else { @@ -849,6 +863,37 @@ pub(super) async fn execute_daemon_invocation( .await } +/// A still-opening project is the mounting refusal the typed CLI re-sends +/// until its deadline. +/// +/// The 500 ms open bound answers "has this route published yet" and leaves +/// the open running. Mapping that miss to [`DaemonInvocationProblem::Unavailable`] +/// republishes `application.surface.unavailable`, which the typed client treats +/// as the answer, so a cold `storage_status` or configuration write fails the +/// moment the bound elapses. Terminal open failures stay on that problem. +fn project_open_refusal_response( + request_id: String, + error: &tracedecay_domain::errors::TraceDecayError, + workflow_application: bool, + git_operation: bool, +) -> DaemonInvocationResponse { + if error_is_project_open_retryable(error) { + return DaemonInvocationResponse::application_problem( + request_id, + tracedecay_contracts::ApplicationProblem::unavailable( + tracedecay_contracts::SafeDiagnostic { + code: tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE.to_owned(), + message: "The project runtime for this operation is still mounting".to_owned(), + }, + ), + ); + } + DaemonInvocationResponse::problem( + request_id, + project_open_problem(error, workflow_application, git_operation), + ) +} + fn project_open_problem( error: &tracedecay_domain::errors::TraceDecayError, workflow_application: bool, @@ -914,6 +959,24 @@ mod workflow_reset_tests { ); } + #[test] + fn warming_project_open_is_a_mounting_refusal_the_client_resends() { + let warming = project_warming_error(Path::new("/tmp/surface-fixture")); + let response = + project_open_refusal_response("request.warming".to_owned(), &warming, false, false); + let tracedecay_daemon_protocol::DaemonInvocationOutcome::ApplicationProblem { problem } = + response.outcome + else { + panic!("warming open must be an application problem, got {response:?}"); + }; + assert_eq!( + problem + .diagnostic() + .map(|diagnostic| diagnostic.code.as_str()), + Some(tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE) + ); + } + #[test] fn failed_project_open_keeps_the_terminal_problem_split() { let failed = tracedecay_domain::errors::TraceDecayError::Config { diff --git a/crates/tracedecay/src/daemon/production_harness.rs b/crates/tracedecay/src/daemon/production_harness.rs index 1f2c6daaaf..a0ffb46bf6 100644 --- a/crates/tracedecay/src/daemon/production_harness.rs +++ b/crates/tracedecay/src/daemon/production_harness.rs @@ -1535,6 +1535,9 @@ mod generation_retention_test; #[cfg(test)] mod configuration_idempotency_journey_test; +#[cfg(test)] +mod configuration_protected_preview_journey_test; + #[cfg(test)] mod read_only_project_open_journey_test; diff --git a/crates/tracedecay/src/daemon/production_harness/configuration_protected_preview_journey_test.rs b/crates/tracedecay/src/daemon/production_harness/configuration_protected_preview_journey_test.rs new file mode 100644 index 0000000000..74b2191aa4 --- /dev/null +++ b/crates/tracedecay/src/daemon/production_harness/configuration_protected_preview_journey_test.rs @@ -0,0 +1,275 @@ +//! Host-facing behavior of `tracedecay_configuration_protected_preview`. +//! +//! The tool is a dry-run: the answer is a redacted plan bound to the revision +//! the caller supplied, and a wrong revision or an invalid change is a typed +//! problem rather than a committed setting. Callers observe that through MCP +//! `tools/call`, which is the path this journey drives. + +use std::collections::BTreeSet; +use std::path::Path; + +use serde_json::{Value, json}; +use tempfile::TempDir; +use tracedecay_contracts::ConfigurationProtectedPreviewRequestV1; +use tracedecay_domain::configuration::{ + AccessRuleId, AuthorityRef, ConfigurationRevisionId, ProtectedChange, RuleEffect, + ScopeAccessRule, ScopeAccessSubjectV1, SourceBindingId, SourceKindV1, +}; +use tracedecay_domain::{CapabilityId, ManifestDigest}; + +use super::journey_test_support::{git, tool_answer}; +use super::*; + +const ACCESS_RULE_ID: &str = "access-rule.preview-cursor-deny"; +const DENIED_CAPABILITY: &str = "capability.work.generate_proposal"; +const ABSENT_BINDING_ID: &str = "source-binding.preview-absent"; +const STALE_REVISION: &str = "configuration.revision.protected-preview-not-current"; + +fn initialize_project(project: &Path) { + std::fs::create_dir_all(project.join("src")).expect("project source"); + std::fs::write(project.join("src/lib.rs"), "pub fn preview_probe() {}\n") + .expect("project source file"); + git(project, &["init", "--quiet"]); +} + +fn preview_arguments(change: &ProtectedChange, revision: &ConfigurationRevisionId) -> Value { + let mut arguments = serde_json::to_value(ConfigurationProtectedPreviewRequestV1 { + change: change.clone(), + expected_revision: revision.clone(), + }) + .expect("protected preview arguments"); + arguments["format"] = json!("json"); + arguments +} + +fn deny_cursor_work(project_id: tracedecay_domain::ProjectId) -> ProtectedChange { + ProtectedChange::UpsertAccessRule( + ScopeAccessRule::new( + AccessRuleId::new(ACCESS_RULE_ID).expect("access rule identity"), + ScopeAccessSubjectV1 { + actor: None, + operation: None, + source_kind: Some(SourceKindV1::Cursor), + }, + AuthorityRef::Project(project_id), + BTreeSet::from([ + CapabilityId::new(DENIED_CAPABILITY).expect("generate proposal capability") + ]), + RuleEffect::Deny, + None, + ) + .expect("deny-only work rule"), + ) +} + +async fn call_preview( + harness: &ProductionProjectCompositionHarnessV1, + project: &Path, + arguments: Value, +) -> (bool, Value) { + let response = harness + .call_tool( + project, + "tracedecay_configuration_protected_preview", + arguments, + ) + .await + .expect("protected preview tools/call"); + tool_answer(&response) +} + +fn assert_redacted_plan( + payload: &Value, + revision: &str, + setting_key: &str, + operation: &str, + before_digest: &str, + after_digest: &str, + hidden: &[&str], +) { + assert_eq!(payload["outcome"]["outcome"], "preview"); + assert_eq!( + payload["outcome"]["value"]["effect_class"], + "configuration_write" + ); + let plan = &payload["outcome"]["value"]["payload"]; + assert_eq!(plan["base_revision_id"], revision); + assert_eq!( + plan["redacted_changes"], + json!([{ + "setting_key": setting_key, + "operation": operation, + "before_digest": before_digest, + "after_digest": after_digest, + }]) + ); + assert_eq!(plan["operation_digest"], after_digest); + assert_eq!(payload["outcome"]["value"]["preview_digest"], after_digest); + assert_eq!( + payload["outcome"]["value"]["preview_id"], plan["plan_id"], + "the preview id the host applies is the plan id" + ); + let plan_id = plan["plan_id"].as_str().expect("plan id"); + assert!( + plan_id.starts_with("configuration.plan.v1."), + "plan id {plan_id} is not a configuration plan" + ); + let created_at = plan["created_at"].as_i64().expect("plan created_at"); + let expires_at = plan["expires_at"].as_i64().expect("plan expires_at"); + assert_eq!( + expires_at - created_at, + 300_000_000, + "a protected preview stays valid for five minutes" + ); + let rendered = serde_json::to_string(payload).expect("preview json"); + for secret in hidden { + assert!( + !rendered.contains(secret), + "preview leaked {secret}: {rendered}" + ); + } +} + +fn assert_problem( + payload: &Value, + kind: &str, + code: &str, + message: &str, + retry: &str, + legal_actions: Value, +) { + assert_eq!(payload["problem"]["kind"], kind, "{payload}"); + assert_eq!(payload["problem"]["code"], code, "{payload}"); + assert_eq!(payload["problem"]["message"], message, "{payload}"); + assert_eq!(payload["problem"]["diagnostic"]["code"], code, "{payload}"); + assert_eq!( + payload["problem"]["diagnostic"]["message"], message, + "{payload}" + ); + assert_eq!(payload["problem"]["retry"], retry, "{payload}"); + assert_eq!( + payload["problem"]["legal_actions"], legal_actions, + "{payload}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn protected_preview_redacts_the_change_and_refuses_stale_or_invalid_input() { + let isolation = TempDir::new().expect("journey isolation"); + let project = isolation.path().join("project"); + initialize_project(&project); + + let harness = ProductionProjectCompositionHarnessV1::open(isolation.path(), [project.clone()]) + .await + .expect("production composition"); + let graph = harness.server(&project).expect("project server").cg().await; + let project_id = graph + .configuration_runtime() + .configuration_target() + .project_id + .clone(); + let current = graph + .configuration_runtime() + .client() + .current() + .await + .expect("current configuration"); + let revision = current.revision_id().clone(); + let before_digest: ManifestDigest = current.snapshot().effective_behavior_digest.clone(); + drop(graph); + + let access_rule = deny_cursor_work(project_id.clone()); + let access_digest = access_rule + .compute_digest() + .expect("access rule digest") + .as_str() + .to_owned(); + let (refused, accepted) = call_preview( + &harness, + &project, + preview_arguments(&access_rule, &revision), + ) + .await; + assert!(!refused, "access-rule preview was refused: {accepted}"); + assert_redacted_plan( + &accepted, + revision.as_str(), + "scope.access_rules.v1", + "access_rule_upsert", + before_digest.as_str(), + &access_digest, + &[ACCESS_RULE_ID, DENIED_CAPABILITY], + ); + + let unbind = ProtectedChange::UnbindSource { + binding_id: SourceBindingId::new(ABSENT_BINDING_ID).expect("binding identity"), + }; + let unbind_digest = unbind + .compute_digest() + .expect("unbind digest") + .as_str() + .to_owned(); + assert_ne!( + access_digest, unbind_digest, + "the two submitted changes must not share a digest" + ); + let (refused, unbound) = + call_preview(&harness, &project, preview_arguments(&unbind, &revision)).await; + assert!(!refused, "unbind preview was refused: {unbound}"); + assert_redacted_plan( + &unbound, + revision.as_str(), + "scope.source_bindings.v1", + "source_unbind", + before_digest.as_str(), + &unbind_digest, + &[ABSENT_BINDING_ID], + ); + + let mut stale = preview_arguments(&access_rule, &revision); + stale["expected_revision"] = json!(STALE_REVISION); + let (refused, conflict) = call_preview(&harness, &project, stale).await; + assert!(refused, "a stale revision must be a tool error: {conflict}"); + assert_problem( + &conflict, + "conflict", + "configuration.conflict", + "The configuration request conflicts with current state", + "after_revalidate", + json!(["refresh"]), + ); + + let mut invalid = preview_arguments(&access_rule, &revision); + invalid["change"]["value"]["capabilities"] = json!([]); + let (refused, rejected) = call_preview(&harness, &project, invalid).await; + assert!( + refused, + "an empty capability set must be a tool error: {rejected}" + ); + assert_problem( + &rejected, + "invalid_request", + "configuration.invalid_request", + "The configuration request is invalid: access rule capabilities must not be empty", + "never", + json!([]), + ); + + let graph = harness.server(&project).expect("project server").cg().await; + let unchanged = graph + .configuration_runtime() + .client() + .current() + .await + .expect("configuration after previews") + .revision_id() + .clone(); + drop(graph); + assert_eq!( + unchanged.as_str(), + revision.as_str(), + "protected preview must not commit a revision" + ); + + harness.shutdown().await; +} diff --git a/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs b/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs index 6415ef3209..b82f5dfe19 100644 --- a/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs @@ -116,10 +116,12 @@ async fn mounted_code_generation_retention_continues_capped_segment_reclamation( ); let graph_replay_pool_root = graph.db().database_path().with_extension("graph-replay"); // The planner probes the generation-store lock and answers - // `GenerationStoreBusy` whenever a writer owns the store; production - // maintenance defers that tick and comes back. This route stays mounted, - // so the pass tail that publishes the edits above can still own the store - // here. Consume the same typed answer instead of reading it as a failure. + // `GenerationStoreBusy` whenever a writer owns the store, and the same + // probe over the graph replay pool answers `GraphReplayPoolBusy`; + // production maintenance defers both and comes back. This route stays + // mounted, so the pass tail that publishes the edits above can still own + // either lock here. Consume the same typed answers instead of reading + // them as failures. let plan = tokio::time::timeout(Duration::from_secs(30), async { loop { match prepare_next_code_generation_retention_cancellable( @@ -129,7 +131,10 @@ async fn mounted_code_generation_retention_continues_capped_segment_reclamation( Some(&graph_replay_pool_root), ) { Ok(plan) => return plan, - Err(CodeGenerationRetentionErrorV1::GenerationStoreBusy) => { + Err( + CodeGenerationRetentionErrorV1::GenerationStoreBusy + | CodeGenerationRetentionErrorV1::GraphReplayPoolBusy, + ) => { tokio::time::sleep(Duration::from_millis(25)).await; } Err(error) => panic!("code generation retention plan: {error:?}"), diff --git a/crates/tracedecay/src/daemon/project_open_owners/code_index_reads/ignored_dependency_admission_tests.rs b/crates/tracedecay/src/daemon/project_open_owners/code_index_reads/ignored_dependency_admission_tests.rs index 946f6bf383..2b72305215 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/code_index_reads/ignored_dependency_admission_tests.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/code_index_reads/ignored_dependency_admission_tests.rs @@ -388,16 +388,13 @@ async fn latest( // Lightweight publication precedes complete-generation seating. Demand // that complete state before using its imports as admission evidence. The // seat is background work behind the scheduler mutex; under a loaded CI - // runner it has taken over 5 s, so the bound is a minute. - // Poll the dashboard projection alone while the owner is busy: the - // query-admission read (`latest_complete_fresh`) leaves a coalesced wake - // behind whenever it finds the worker holding the scheduler with an - // expired proof, and polling it every 25 ms re-armed a no-op pass faster - // than the ladder could settle to `Fresh` (CI run 35419627712: one - // minute of `Verifying`, then 0.3 s on the retry). Read the generation - // only once the ladder has settled. + // runner it has taken over 5 s, so the bound is a minute. Polling the + // query read is safe: a read that finds the owner holding the scheduler + // does not schedule the successor the dashboard would project as + // `Verifying`. tokio::time::timeout(Duration::from_mins(1), async { loop { + let _ = registry.latest_complete_fresh(project_root).await; if registry .dashboard_freshness(project_root) .await diff --git a/crates/tracedecay/src/daemon/store_runtime_tests.rs b/crates/tracedecay/src/daemon/store_runtime_tests.rs index 8799d13b61..c7ec20dfad 100644 --- a/crates/tracedecay/src/daemon/store_runtime_tests.rs +++ b/crates/tracedecay/src/daemon/store_runtime_tests.rs @@ -1206,7 +1206,7 @@ async fn retained_runtime_ledger_replays_during_bounded_background_convergence() .observation_store() .advance_source_cursor(rereasoned_advance) .await - .expect("classify a re-reasoned retained cursor while convergence is pending"), + .expect("a later reason does not unseat the owned frontier"), CursorAdvanceOutcome::ExactDuplicate ); @@ -1261,7 +1261,7 @@ async fn retained_runtime_ledger_replays_during_bounded_background_convergence() .get::(0) .expect("decode committed cursor effect count"), 2, - "the retained replay and collision must not create another cursor effect" + "the retained replay and the later reason must not create another cursor effect" ); let mut receipts = snapshot .query( diff --git a/crates/tracedecay/src/daemon/tests/invocation_ownership.rs b/crates/tracedecay/src/daemon/tests/invocation_ownership.rs index 1e0480a268..bb453ae7d7 100644 --- a/crates/tracedecay/src/daemon/tests/invocation_ownership.rs +++ b/crates/tracedecay/src/daemon/tests/invocation_ownership.rs @@ -421,7 +421,10 @@ async fn retained_invocation_while_owners_mount_is_retryable_not_unmounted() { let diagnostic = problem .diagnostic() .expect("a mounting retained owner carries a diagnostic"); - assert_eq!(diagnostic.code, "application.surface.unavailable"); + assert_eq!( + diagnostic.code, + tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE + ); assert!( !diagnostic .message diff --git a/crates/tracedecay/tests/common/mod.rs b/crates/tracedecay/tests/common/mod.rs index 201a7a2b38..08bed31dcc 100644 --- a/crates/tracedecay/tests/common/mod.rs +++ b/crates/tracedecay/tests/common/mod.rs @@ -17,6 +17,8 @@ use std::os::unix::fs::PermissionsExt; use std::os::unix::process::CommandExt; use std::path::{Path, PathBuf}; use std::process::{Child, Command, ExitStatus, Output, Stdio}; +#[cfg(unix)] +use std::sync::{Mutex, PoisonError}; use std::time::{Duration, Instant}; use serde_json::Value; @@ -657,6 +659,14 @@ pub fn http_agent_with_timeout(timeout: Duration) -> ureq::Agent { /// panic while the child is still running, `Drop` force-stops and reaps it. pub struct TestChildProcess { child: Child, + /// Whether this child has been waited on. A reaped pid belongs to the + /// kernel again, so it must never be used to address a process group. + reaped: bool, + /// Path of a Unix socket this child published. Released after the process + /// group is reaped so a descendant that still holds the listen descriptor + /// cannot keep the path accepting. + #[cfg(unix)] + release_socket: Option, } /// Daemon-specific name retained for test fixtures that keep a daemon alive. @@ -664,7 +674,51 @@ pub type DaemonProcess = TestChildProcess; impl TestChildProcess { pub fn new(child: Child) -> Self { - Self { child } + Self { + child, + reaped: false, + #[cfg(unix)] + release_socket: None, + } + } + + /// Unlink the socket this child published once it has been reaped. + /// + /// `process_group(0)` makes the child a group leader. Stopping only that + /// pid leaves descendants that still hold the listen socket. Group-kill + /// closes those descriptors; unlinking the path is what makes a later + /// `connect` fail even if the kernel has not finished the last close. + /// + /// Recording claims the path: a restart journey reassigns its handle + /// (`daemon = spawn(..)`), so the successor is already publishing when + /// the predecessor is dropped, and only the current publisher may unlink. + /// File identity is not enough for that - the successor's socket routinely + /// lands on the inode the predecessor's shutdown just freed. + #[cfg(unix)] + pub fn release_socket_on_stop(&mut self, path: PathBuf) { + claim_published_socket(&path, self.child.id()); + self.release_socket = Some(path); + } + + #[cfg(unix)] + fn release_recorded_socket(&mut self) { + let Some(path) = self.release_socket.take() else { + return; + }; + if !release_published_socket_claim(&path, self.child.id()) { + // A successor publishes here now; its socket is not ours to unlink. + return; + } + match std::fs::remove_file(&path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + panic!( + "failed to release daemon socket '{}': {error}", + path.display() + ) + } + } } pub fn id(&self) -> u32 { @@ -676,7 +730,9 @@ impl TestChildProcess { } pub fn try_wait(&mut self) -> std::io::Result> { - self.child.try_wait() + let status = self.child.try_wait()?; + self.reaped |= status.is_some(); + Ok(status) } pub fn wait_for_exit(&mut self, timeout: Duration) -> std::io::Result> { @@ -752,9 +808,15 @@ impl TestChildProcess { /// Force-stops the daemon and reaps its process before returning. /// /// `Child::kill` maps to `SIGKILL` on Unix and the platform termination - /// primitive elsewhere, keeping fault-injection tests portable. + /// primitive elsewhere, keeping fault-injection tests portable. On Unix + /// the child's process group is signaled first, then the published socket + /// path is unlinked. pub fn kill_and_wait(&mut self) -> std::io::Result { - terminate_and_reap(&mut self.child) + let status = terminate_and_reap(&mut self.child, !self.reaped); + self.reaped = true; + #[cfg(unix)] + self.release_recorded_socket(); + status } fn drain_stderr(&mut self) { @@ -778,32 +840,37 @@ impl TestChildProcess { impl Drop for TestChildProcess { fn drop(&mut self) { - let _ = terminate_and_reap(&mut self.child); + let _ = terminate_and_reap(&mut self.child, !self.reaped); + self.reaped = true; + #[cfg(unix)] + self.release_recorded_socket(); } } -/// PID-directed stop: survives `process_group(0)` / `setsid` detachment. +/// Stop a child that was detached with `process_group(0)`. /// -/// The child is the leader of its own group. Killing only that pid leaves -/// helper children that still hold the listen socket, so the next spawn -/// observes a connectable daemon after this process has already been reaped. -fn terminate_and_reap(child: &mut Child) -> std::io::Result { - if let Ok(Some(status)) = child.try_wait() { - return Ok(status); +/// The child is the leader of its own group. `SIGKILL` of that pid alone +/// leaves descendants in the group. Those descendants keep any descriptor they +/// inherited, including a listen socket, so the path stays connectable after +/// `wait` returns. Signaling the group first closes those descriptors; the +/// leader kill still covers a child whose `setpgid` has not run yet. +/// +/// `signal_group` must be false once this child has been waited on: a reaped +/// pid is the kernel's to reissue, so negating it could address a process +/// group this harness never created. +fn terminate_and_reap(child: &mut Child, signal_group: bool) -> std::io::Result { + // Signal the group before reaping. A leader that has already exited still + // names the group while it is an unreaped zombie; returning on `try_wait` + // first would leave descendants holding the listen socket. + #[cfg(unix)] + if signal_group { + signal_child_process_group(child.id()); } + #[cfg(not(unix))] + let _ = signal_group; - #[cfg(unix)] - { - let pid = child.id(); - if pid != 0 { - // SAFETY: `pid` is this live child. Negating it targets the - // process group `process_group(0)` created with that pid as - // leader. ESRCH is ignored: setpgid may not have run yet, and the - // pid kill below still stops the leader. - unsafe { - libc::kill(-(pid as i32), libc::SIGKILL); - } - } + if let Ok(Some(status)) = child.try_wait() { + return Ok(status); } if let Err(kill_err) = child.kill() { @@ -816,6 +883,50 @@ fn terminate_and_reap(child: &mut Child) -> std::io::Result { child.wait() } +/// The child pid currently publishing each recorded socket path. +#[cfg(unix)] +static PUBLISHED_SOCKETS: Mutex> = Mutex::new(Vec::new()); + +#[cfg(unix)] +fn claim_published_socket(path: &Path, pid: u32) { + let mut claims = PUBLISHED_SOCKETS + .lock() + .unwrap_or_else(PoisonError::into_inner); + claims.retain(|(claimed, _)| claimed != path); + claims.push((path.to_path_buf(), pid)); +} + +/// True when `pid` is still the publisher of `path`, dropping the claim. +#[cfg(unix)] +fn release_published_socket_claim(path: &Path, pid: u32) -> bool { + let mut claims = PUBLISHED_SOCKETS + .lock() + .unwrap_or_else(PoisonError::into_inner); + let Some(index) = claims + .iter() + .position(|(claimed, owner)| claimed == path && *owner == pid) + else { + return false; + }; + claims.swap_remove(index); + true +} + +#[cfg(unix)] +fn signal_child_process_group(pid: u32) { + let Ok(pid) = i32::try_from(pid) else { + return; + }; + if pid == 0 { + return; + } + // SAFETY: `pid` is the spawned child's id. Negating it addresses the + // process group `process_group(0)` created with that pid as leader. + // `ESRCH` is ignored: the child may not be a group leader, and the pid + // kill in `terminate_and_reap` still stops it. + let _ = unsafe { libc::kill(-pid, libc::SIGKILL) }; +} + /// Detach a test child from the test process group. /// /// Nextest (and other harness timeouts) signal the test's process group. @@ -1145,7 +1256,8 @@ fn spawn_tracedecay_daemon_process( // daemon, which is what `init_project_fixture` journeys (spawn, init, drop, // spawn again) hit on a loaded runner. Wait a bounded time for the endpoint // to stop accepting; a daemon that keeps accepting still fails with the - // same refusal. + // same refusal. The group signal in `terminate_and_reap` is what makes the + // endpoint go quiet; this wait only covers the kernel's leftover. poll_until( Instant::now() + PREDECESSOR_DAEMON_VACATE_TIMEOUT, Duration::from_millis(25), @@ -1187,6 +1299,8 @@ fn spawn_tracedecay_daemon_process( detach_from_test_process_group(&mut command); let child = command.spawn().expect("tracedecay daemon should start"); let mut daemon = DaemonProcess::new(child); + #[cfg(unix)] + daemon.release_socket_on_stop(socket_path.clone()); let deadline = Instant::now() + Duration::from_secs(10); poll_until( diff --git a/crates/tracedecay/tests/daemon_suite/advanced_workflow_journey/daemon_fixture.rs b/crates/tracedecay/tests/daemon_suite/advanced_workflow_journey/daemon_fixture.rs index d0bbc79f28..069897ebb7 100644 --- a/crates/tracedecay/tests/daemon_suite/advanced_workflow_journey/daemon_fixture.rs +++ b/crates/tracedecay/tests/daemon_suite/advanced_workflow_journey/daemon_fixture.rs @@ -105,6 +105,8 @@ pub(super) fn spawn_project_daemon(home: &Path, project: &Path) -> common::Daemo .spawn() .expect("advanced workflow daemon should start"); let mut daemon = common::DaemonProcess::new(child); + #[cfg(unix)] + daemon.release_socket_on_stop(common::daemon_socket_path(home)); let daemon_pid = u64::from(daemon.id()); let deadline = Instant::now() + Duration::from_secs(120); loop { diff --git a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs index bc1388df7f..28a66cd569 100644 --- a/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs +++ b/crates/tracedecay/tests/daemon_suite/code_index_ignored_dependencies_test/flight_tests.rs @@ -1,7 +1,10 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Condvar, Mutex}; -use tracedecay_code_index::production::CodeIndexProductionErrorV1; +use tracedecay_code_index::production::{ + CodeIndexProductionErrorV1, CodeIndexPublicationStoreErrorV1, +}; +use tracedecay_code_index_retention::code_index_generations::try_acquire_code_generation_store_lock; use super::*; @@ -148,15 +151,50 @@ export function GenerationAnchor(value: PublicWidget) { return value; } } fn assert_publication_error(error: CodeIndexSchedulerErrorV1) { + let CodeIndexSchedulerErrorV1::Production(CodeIndexProductionErrorV1::Publication( + CodeIndexPublicationStoreErrorV1::CorruptionResetRequired(detail), + )) = &error + else { + panic!( + "coalesced failure must stay in the scheduler publication family, not an EISDIR misclass: {error:?}" + ); + }; + assert!( + detail.contains("not a regular file"), + "a directory pointer slot is publication corruption, got {detail}" + ); assert!( - matches!( - error, - CodeIndexSchedulerErrorV1::Production(CodeIndexProductionErrorV1::Publication(_)) - ), - "coalesced failure must preserve the production publication error family" + !detail.contains("Is a directory") && !detail.contains("os error 21"), + "publication corruption must not carry the raw EISDIR OS error: {detail}" ); } +/// Hold the only background permit once no pass is in flight. +/// +/// Text seating keeps `reconcile_in_progress` after it drops the scheduler +/// mutex, and that pass can still rename a valid active pointer. A truncated +/// pointer written in that window is not a closed fault. Occupying the permit +/// while the owner has not entered its pass stops that rewrite. +async fn hold_idle_background_admission( + registry: &CodeIndexSchedulerRegistryV1, +) -> tokio::sync::OwnedSemaphorePermit { + let admission = registry.background_reconcile_admission(); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + if registry.memory_stats().await.reconciling_worktrees == 0 + && let Ok(permit) = admission.clone().try_acquire_owned() + && registry.memory_stats().await.reconciling_worktrees == 0 + { + return permit; + } + assert!( + std::time::Instant::now() <= deadline, + "background reconcile did not go idle before publication fault injection" + ); + tokio::time::sleep(Duration::from_millis(2)).await; + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn aborted_flight_owner_wakes_follower_and_allows_a_fresh_owner() { let fixture = fixture(); @@ -242,6 +280,8 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { let registry = Arc::new(mount(fixture.path(), &store, 1).await); let baseline = latest(®istry, fixture.path()).await; let request = request_for(&baseline, "pkg"); + let idle_admission = hold_idle_background_admission(®istry).await; + registry.clear_pending_wake_for_scope(&request.scope).await; let hold = SchedulerHold::acquire(®istry, fixture.path()).await; let (owner_control, owner_entered) = BlockingNthControl::new(4); @@ -283,22 +323,12 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { &fixture.path().canonicalize().expect("canonical fixture"), ); let pointer_path = scoped_store.join("active-code-generation-v1.json"); - // Every production writer of the active pointer reads it, edits it in - // memory and renames a temporary over it while holding the exclusive - // generation-store lock. Corrupting the file without that lock races an - // in-flight read-modify-write whose rename then restores a valid pointer, - // and this owner publishes instead of failing closed. The racer is the - // background pass tail: it releases the background admission permit this - // owner then takes (registry/mount.rs, "release the background admission - // permit before HeadOpening / graph work") and keeps attaching the - // generation's text artifact afterwards, so neither the held admission - // nor the held scheduler mutex proves the store is quiet. Taking the - // store lock does: being granted it means no writer is mid-transaction, - // and any writer that starts after it is released reads the corruption - // under the lock and refuses instead of overwriting it. + // Writers rename a temporary over the active pointer while holding the + // generation-store lock. A truncated file is not a closed fault: a pass + // that already read a valid pointer can rename it back. A directory cannot + // be renamed over. Taking the lock first means no writer is mid-transaction + // when the slot stops being a regular file. let pointer_bytes = { - use tracedecay_code_index_retention::code_index_generations::try_acquire_code_generation_store_lock; - let store_lock = tokio::time::timeout(Duration::from_secs(5), async { loop { if let Some(lock) = try_acquire_code_generation_store_lock(&scoped_store) @@ -312,10 +342,16 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { .await .expect("no generation-store writer is mid-transaction"); let pointer_bytes = std::fs::read(&pointer_path).expect("read active pointer"); - std::fs::write(&pointer_path, b"{").expect("corrupt active pointer"); + // `rename(2)` replaces a truncated file with a valid pointer. A + // directory cannot be renamed over, so the fault stays closed. The + // scheduler must report publication corruption, not the EISDIR that + // read and rename return for that directory. + std::fs::remove_file(&pointer_path).expect("remove active pointer"); + std::fs::create_dir(&pointer_path).expect("replace active pointer with a directory"); drop(store_lock); pointer_bytes }; + drop(idle_admission); owner_control.release(); hold.release(); @@ -331,7 +367,12 @@ async fn coalesced_publication_failure_preserves_the_scheduler_error_family() { .expect_err("follower publication fails closed"); assert_publication_error(owner_error); assert_publication_error(follower_error); + assert!( + pointer_path.is_dir(), + "publication must not replace a directory pointer it did not observe" + ); + std::fs::remove_dir_all(&pointer_path).expect("remove faulted pointer node"); std::fs::write(pointer_path, pointer_bytes).expect("restore active pointer"); registry.shutdown().await; } diff --git a/crates/tracedecay/tests/daemon_suite/main.rs b/crates/tracedecay/tests/daemon_suite/main.rs index 6339fab5c6..8e86e9cd8d 100644 --- a/crates/tracedecay/tests/daemon_suite/main.rs +++ b/crates/tracedecay/tests/daemon_suite/main.rs @@ -31,6 +31,7 @@ mod indexing_lifecycle_test; mod invocation_observability; mod invocation_primitives; #[cfg(unix)] +mod socket_lifecycle_test; #[cfg(unix)] mod stale_client_resilience_test; mod workflow_handoff_test; diff --git a/crates/tracedecay/tests/daemon_suite/socket_lifecycle_test.rs b/crates/tracedecay/tests/daemon_suite/socket_lifecycle_test.rs new file mode 100644 index 0000000000..bd2f0f1dbc --- /dev/null +++ b/crates/tracedecay/tests/daemon_suite/socket_lifecycle_test.rs @@ -0,0 +1,104 @@ +//! Process-group stop must release a listen socket held by a descendant. +//! +//! `process_group(0)` makes the spawned child its own group leader. Killing +//! only that pid leaves the descendant that inherited the listen descriptor, +//! and the path stays connectable after `wait` returns. + +use std::os::unix::net::UnixStream; +use std::os::unix::process::CommandExt; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +use crate::common::{TestChildProcess, poll_until}; + +/// How long the released descendant has to finish dying. +/// +/// The group signal is delivered to a process the harness cannot `wait` on - +/// the descendant is reparented, not a child - so its descriptors close when +/// the kernel finishes tearing it down, not when the leader's `wait` returns. +/// A descendant that was never signaled keeps accepting past this deadline, +/// which is the regression this proof exists to catch. +const DESCENDANT_RELEASE_TIMEOUT: Duration = Duration::from_secs(10); + +const HOLDER: &str = r#" +import os, socket, time +path = os.environ["TRACEDECAY_TEST_SOCKET_PATH"] +listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) +listener.bind(path) +listener.listen(1) +os.fork() +while True: + time.sleep(60) +"#; + +#[test] +fn group_stop_releases_an_inherited_listen_socket() { + let scratch = tempfile::tempdir().expect("socket scratch"); + let socket = scratch.path().join("daemon.sock"); + let mut command = Command::new("python3"); + command + .arg("-c") + .arg(HOLDER) + .env("TRACEDECAY_TEST_SOCKET_PATH", &socket) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + command.process_group(0); + let child = command.spawn().expect("spawn socket holder"); + // Do not record a socket path: connect must fail because the group is + // dead, not because the path was unlinked. + let mut holder = TestChildProcess::new(child); + + let ready_deadline = Instant::now() + Duration::from_secs(5); + while UnixStream::connect(&socket).is_err() { + assert!( + Instant::now() < ready_deadline, + "holder did not bind {}", + socket.display() + ); + if holder.try_wait().expect("holder status").is_some() { + panic!("socket holder exited before binding"); + } + std::thread::sleep(Duration::from_millis(20)); + } + + holder.kill_and_wait().expect("reap socket holder group"); + assert!( + socket.exists(), + "this proof must not delete the socket path" + ); + poll_until( + Instant::now() + DESCENDANT_RELEASE_TIMEOUT, + Duration::from_millis(20), + || UnixStream::connect(&socket).is_err().then_some(()), + || { + format!( + "process-group stop must release the inherited listen socket at {}", + socket.display() + ) + }, + ); +} + +#[test] +fn stop_unlinks_the_socket_path_the_child_published() { + let scratch = tempfile::tempdir().expect("socket scratch"); + let socket = scratch.path().join("daemon.sock"); + std::os::unix::net::UnixListener::bind(&socket).expect("bind socket"); + let mut command = Command::new("python3"); + command + .arg("-c") + .arg("import time; time.sleep(60)") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + command.process_group(0); + let child = command.spawn().expect("spawn sleeper"); + let mut sleeper = TestChildProcess::new(child); + sleeper.release_socket_on_stop(socket.clone()); + drop(sleeper); + assert!( + !socket.exists(), + "stopping the child must unlink the socket path it published" + ); +} diff --git a/crates/tracedecay/tests/mcp_suite/git_correlation_test.rs b/crates/tracedecay/tests/mcp_suite/git_correlation_test.rs index 672dc3a9b4..45b51eb751 100644 --- a/crates/tracedecay/tests/mcp_suite/git_correlation_test.rs +++ b/crates/tracedecay/tests/mcp_suite/git_correlation_test.rs @@ -160,8 +160,9 @@ async fn host_call(server: &McpServer, mut args: Value) -> HostCall { .unwrap_or_else(|| { panic!("tracedecay_sessions_for returned no JSON content: {response}") }); - if text.pointer("/problem/code").and_then(Value::as_str) - == Some("application.surface.unavailable") + let code = text.pointer("/problem/code").and_then(Value::as_str); + if code == Some(tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE) + || code == Some("application.surface.unavailable") { tokio::time::sleep(std::time::Duration::from_millis(100)).await; continue; diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test.rs index d7ca7928f5..65d1c4cb4b 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test.rs @@ -95,6 +95,7 @@ mod project_search_behavior_test; mod rank_behavior_test; #[cfg(all(feature = "test-transport", unix))] mod release_placement_test; +mod remote_status_test; mod rename_preview_test; #[cfg(feature = "test-transport")] mod rename_symbol_test; diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/graph_query_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/graph_query_test.rs index d5bb7ea230..2ea3be05ea 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/graph_query_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/graph_query_test.rs @@ -104,6 +104,10 @@ pub struct PlainValue; .await } +fn clone_family_lane_still_publishing(error: &str) -> bool { + error.contains("generation_unverified") +} + async fn shutdown_graph_fixture(fixture: GraphQueryFixture) { fixture.production.harness.shutdown().await; } @@ -1577,28 +1581,57 @@ async fn redundancy_pull_request_scope_shares_one_budget_and_resumes_changed_fam .server(fixture.project_root()) .expect("production graph-query server"); warm_code_index_search(&server, "generation_bump").await; - let stale = call_production_tool( - &fixture, - "tracedecay_redundancy", - json!({ - "project_id": project_id, - "repository_id": repository_id, - "match_classes": ["conservative_exact"], - "scope": scope, - "include_generated_paths": true, - "family_limit": 10, - "member_limit": 10, - "work_limit": 4, - "cursor": stale_cursor, - }), - None, - None, - ) - .await - .expect_err("a prior-generation pull-request cursor must be stale"); + let stale_args = json!({ + "project_id": project_id, + "repository_id": repository_id, + "match_classes": ["conservative_exact"], + "scope": scope, + "include_generated_paths": true, + "family_limit": 10, + "member_limit": 10, + "work_limit": 4, + "cursor": stale_cursor, + }); + // Search lane coverage can seal while clone-family publication is still + // retiring the previous generation. That window answers `search_failed` + // or `generation_unverified`; the stale cursor's terminal is + // `generation_unavailable` once the successor artifact can be read. + let mut last = String::new(); + for _ in 0..40 { + match call_production_tool( + &fixture, + "tracedecay_redundancy", + stale_args.clone(), + None, + None, + ) + .await + { + Err(error) => { + let rendered = error.to_string(); + if rendered.contains("generation_unavailable") { + last = rendered; + break; + } + if clone_family_lane_still_publishing(&rendered) { + last = rendered; + tokio::time::sleep(Duration::from_millis(250)).await; + continue; + } + panic!("stale pull-request cursor failed closed: {rendered}"); + } + Ok(value) => { + last = format!( + "successor still served the prior cursor: {}", + extract_text(&value.value) + ); + tokio::time::sleep(Duration::from_millis(250)).await; + } + } + } assert!( - stale.to_string().contains("generation_unavailable"), - "{stale}" + last.contains("generation_unavailable"), + "prior-generation pull-request cursor must be stale, last={last}" ); shutdown_graph_fixture(fixture).await; diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_describe_behavior.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_describe_behavior.rs index 04b77576fd..324bc17c0e 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_describe_behavior.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_describe_behavior.rs @@ -247,6 +247,9 @@ async fn describe_raw(server: &Arc, arguments: Value) -> Value { } fn session_document(node_id: &str, payload_ref: &str) -> Value { + let external_placeholder = format!( + "[Externalized LCM ingest payload: kind=tool_result; field=content; chars=320040; bytes=320040; ref={payload_ref}]" + ); json!({ "description": { "external_payload": null, @@ -257,13 +260,13 @@ fn session_document(node_id: &str, payload_ref: &str) -> Value { "raw_message_count": 2, "raw_messages": [ { - "content_preview": "", + "content_preview": SOURCE_BODY, "content_range": { - "limit": 0, + "limit": SOURCE_BODY.len(), "offset": 0, - "returned_chars": 0, + "returned_chars": SOURCE_BODY.len(), "total_chars": SOURCE_BODY.len(), - "truncated": true + "truncated": false }, "message_id": SOURCE_ID, "payload_ref": null, @@ -272,12 +275,12 @@ fn session_document(node_id: &str, payload_ref: &str) -> Value { "store_id": 1 }, { - "content_preview": "", + "content_preview": external_placeholder, "content_range": { - "limit": 0, + "limit": external_placeholder.len(), "offset": 0, - "returned_chars": 0, - "total_chars": 180, + "returned_chars": external_placeholder.len(), + "total_chars": 320_040, "truncated": true }, "message_id": TOOL_ID, @@ -298,7 +301,7 @@ fn session_document(node_id: &str, payload_ref: &str) -> Value { "depth": 0, "node_id": node_id, "source_count": 1, - "summary_preview": "" + "summary_preview": SUMMARY } ], "target": "session" @@ -402,7 +405,9 @@ fn external_payload_document(payload_ref: &str, content_hash: &str) -> Value { "byte_count": 320_040, "char_count": 320_040, "content_hash": content_hash, - "content_preview": "", + "content_preview": format!( + "[Externalized LCM ingest payload: kind=tool_result; field=content; chars=320040; bytes=320040; ref={payload_ref}]" + ), "created_at": "", "kind": "tool_result", "message_id": TOOL_ID, diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_test.rs index a39c62135b..3769ae4fdb 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_test.rs @@ -65,7 +65,8 @@ async fn lcm_session_handlers_expose_bounded_read_apis_and_placeholders() { let (cg, _env) = init_test_project(dir.path()).await; let full_text = format!("orchard dispatch {}", "external-payload-body ".repeat(220)); let projection = - seed_temporal_lcm_session_message(&cg, "lcm-session", "lcm-message", full_text, 1).await; + seed_temporal_lcm_session_message(&cg, "lcm-session", "lcm-message", full_text.clone(), 1) + .await; let temporal_db = open_active_project_session_db(&cg).await; activate_test_temporal_generation(&temporal_db, "lcm-session", vec![projection]).await; let db = open_active_project_session_db(&cg).await; @@ -286,10 +287,21 @@ async fn lcm_session_handlers_expose_bounded_read_apis_and_placeholders() { "{described_payload}" ); assert_eq!(described_payload["description"]["raw_message_count"], 1); + let preview = described_payload["description"]["raw_messages"][0]["content_preview"] + .as_str() + .expect("describe preview"); assert!( - described_payload["description"]["raw_messages"][0] - .get("content_preview") - .is_some() + preview.starts_with("orchard dispatch"), + "describe returned an empty preview: {preview:?}" + ); + assert!( + preview.chars().count() < full_text.chars().count(), + "describe echoed the full payload body" + ); + assert_eq!( + described_payload["description"]["raw_messages"][0]["content_range"]["total_chars"], + full_text.chars().count() as u64, + "describe must name the captured message length, not the preview stub" ); assert!( described_payload["description"]["raw_messages"][0] @@ -742,9 +754,12 @@ async fn lcm_describe_supports_summary_node_and_external_payload_targets() { payload_payload["description"]["external_payload"]["payload_ref"], payload_ref ); - assert_eq!( - payload_payload["description"]["external_payload"]["content_preview"], - "" + let payload_preview = payload_payload["description"]["external_payload"]["content_preview"] + .as_str() + .unwrap_or_else(|| panic!("payload describe preview missing: {payload_payload}")); + assert!( + payload_preview.contains(payload_ref.as_str()), + "payload describe must return the stored placeholder: {payload_preview:?}" ); assert_eq!(payload_payload["grain"], "occurrence"); assert_eq!(payload_payload["state"], "available"); diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/remote_status_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/remote_status_test.rs new file mode 100644 index 0000000000..078dd96375 --- /dev/null +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/remote_status_test.rs @@ -0,0 +1,164 @@ +//! Real `tools/call` coverage for `tracedecay_remote_status`. +//! +//! The production daemon mounts the Remote Brain reader. With no listener and +//! no registered node, that reader is `unconfigured`. A direct server never +//! installs the reader, so the same call is `unavailable`. Neither outcome is +//! an empty success or a semantic tool error. +//! +//! The configured (`observed`) plane is proved at the two seams that can mount +//! one cheaply: `daemon::remote_protocol_tests` provisions a real node and +//! serving listener against the session runtime registry, and +//! `mcp::tools::handlers::info::remote_status_dispatch_tests` drives an +//! observed reader through dispatch. This suite owns the transport boundary +//! those two do not cross. + +use std::path::PathBuf; +use std::process::Command; + +use serde_json::{Value, json}; +use tracedecay::daemon::ProductionProjectCompositionHarnessV1; + +use crate::common; +use crate::fixture; +use crate::mcp_server_test::support::{ + jsonrpc_request, response_with_id, run_server_with_messages, setup_server, successful_tool_text, +}; +use crate::support::{TestTempDir, test_temp_dir}; + +const UNCONFIGURED_JSON: &str = r#"{"kind":"unconfigured"}"#; +const UNAVAILABLE_JSON: &str = r#"{"kind":"unavailable"}"#; +const UNCONFIGURED_MARKDOWN: &str = "**kind:** unconfigured\n"; +const UNAVAILABLE_MARKDOWN: &str = "**kind:** unavailable\n"; + +struct MountedDaemon { + harness: ProductionProjectCompositionHarnessV1, + project: PathBuf, + _isolation: TestTempDir, +} + +async fn mount_daemon_without_remote_plane() -> MountedDaemon { + let isolation = test_temp_dir(); + let project = isolation.path().join("project"); + std::fs::create_dir_all(&project).expect("remote-status project directory"); + fixture::write_indexed_fixture_sources(&project); + for args in [ + vec!["init", "-q"], + vec!["add", "."], + vec![ + "-c", + "user.name=TraceDecay Test", + "-c", + "user.email=tracedecay@example.invalid", + "commit", + "-qm", + "remote status fixture", + ], + ] { + let status = Command::new(common::git_program()) + .args(args) + .current_dir(&project) + .status() + .expect("git"); + assert!(status.success(), "git must succeed for {project:?}"); + } + let harness = Box::pin( + ProductionProjectCompositionHarnessV1::open_for_session_retrieval( + isolation.path(), + [project.clone()], + ), + ) + .await + .expect("production composition"); + MountedDaemon { + harness, + project, + _isolation: isolation, + } +} + +fn status_text(result: &Value) -> &str { + let content = result["content"] + .as_array() + .unwrap_or_else(|| panic!("remote status returned no content array: {result}")); + assert_eq!( + content.len(), + 1, + "remote status must not attach banners or token footers: {result}" + ); + assert!( + result.get("isError").is_none(), + "a typed remote-status read is not a semantic tool error: {result}" + ); + content[0]["text"] + .as_str() + .unwrap_or_else(|| panic!("remote status text content missing: {result}")) +} + +async fn daemon_status(mounted: &MountedDaemon, arguments: Value) -> Value { + let response = mounted + .harness + .call_tool(&mounted.project, "tracedecay_remote_status", arguments) + .await + .expect("production tools/call"); + assert!( + response.error.is_none(), + "production remote status must succeed: {:?}", + response.error + ); + response + .result + .unwrap_or_else(|| panic!("production remote status missing result")) +} + +#[tokio::test] +async fn production_daemon_reports_unconfigured_remote_plane() { + let mounted = mount_daemon_without_remote_plane().await; + + let markdown = daemon_status(&mounted, json!({})).await; + assert_eq!(status_text(&markdown), UNCONFIGURED_MARKDOWN); + + let json_result = daemon_status(&mounted, json!({"format": "json"})).await; + assert_eq!(status_text(&json_result), UNCONFIGURED_JSON); +} + +#[tokio::test] +async fn direct_server_reports_unmounted_remote_authority() { + let (server, _dir) = setup_server().await; + let responses = run_server_with_messages( + server, + vec![ + jsonrpc_request( + json!(1), + "tools/call", + json!({ + "name": "tracedecay_remote_status", + "arguments": {} + }), + ), + jsonrpc_request( + json!(2), + "tools/call", + json!({ + "name": "tracedecay_remote_status", + "arguments": {"format": "json"} + }), + ), + ], + ) + .await; + + let markdown = response_with_id(&responses, json!(1)); + assert_eq!( + successful_tool_text(&markdown, "markdown remote status"), + UNAVAILABLE_MARKDOWN + ); + let json_response = response_with_id(&responses, json!(2)); + assert_eq!( + successful_tool_text(&json_response, "json remote status"), + UNAVAILABLE_JSON + ); + assert!( + json_response["result"].get("isError").is_none(), + "an unmounted remote authority is a typed read, not a tool error: {json_response}" + ); +} diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/session_search_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/session_search_test.rs index 28f292af5f..ff713e337b 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/session_search_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/session_search_test.rs @@ -674,6 +674,39 @@ async fn production_codex_hook_ingest_survives_message_search_reopen() { expanded["expansion"]["raw_message"]["message_id"], message_id, "{expanded}" ); + let described = call_production_tool( + &harness, + &project, + "tracedecay_lcm_describe", + json!({ + "provider": "codex", + "session_id": session_id, + "target": {"kind": "session"}, + "format": "json" + }), + ) + .await; + let captured = "Find the cobalt orchard scheduler migration"; + let overview = described["description"]["raw_messages"] + .as_array() + .and_then(|messages| { + messages + .iter() + .find(|message| message["message_id"] == message_id) + }) + .unwrap_or_else(|| panic!("describe omitted the captured prompt: {described}")); + assert_eq!( + overview["content_range"]["total_chars"], + captured.chars().count() as u64, + "{overview}" + ); + let preview = overview["content_preview"] + .as_str() + .unwrap_or_else(|| panic!("describe preview missing: {overview}")); + assert!( + preview.contains("cobalt orchard"), + "describe preview was empty: {preview:?}" + ); harness.shutdown().await; diff --git a/crates/tracedecay/tests/product_surface_suite/main.rs b/crates/tracedecay/tests/product_surface_suite/main.rs index 9f0d827907..d95e6c0931 100644 --- a/crates/tracedecay/tests/product_surface_suite/main.rs +++ b/crates/tracedecay/tests/product_surface_suite/main.rs @@ -11,6 +11,9 @@ mod catalog_composition_contract; mod git_intelligence_regression; mod host_bundle_acceptance; mod native_integration_surface_mount; +// See `runtime_acceptance_suite`: the evaluator library is opt-in so transport +// tests do not compile the eval-only lexical projection. +#[cfg(feature = "search-eval")] mod packaged_search_evaluator; mod profile_backup_rehearsal_test; mod verified_profile_backup; diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/advisory_runtime_acceptance.rs b/crates/tracedecay/tests/runtime_acceptance_suite/advisory_runtime_acceptance.rs index d40e739a48..80e2410830 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/advisory_runtime_acceptance.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/advisory_runtime_acceptance.rs @@ -1100,37 +1100,70 @@ async fn packaged_host_ingest_delivers_a_registered_advisory_cycle() { "format": "json", }) .to_string(); - let stop_output = common::tracedecay_command_with_home(environment.home()) - .args([ - "tool", - "--project", - project_arg.as_str(), - "tracedecay_hook_runtime", - "--args", - stop_args.as_str(), - "--json", - ]) - .current_dir(&project) - .output() - .expect("invoke registered daemon stop path"); - assert!( - stop_output.status.success(), - "registered daemon stop ingest failed\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&stop_output.stdout), - String::from_utf8_lossy(&stop_output.stderr) - ); - let stop_response: Value = - serde_json::from_slice(&stop_output.stdout).expect("registered daemon stop response"); - let stop_payload: Value = serde_json::from_str( - stop_response["content"][0]["text"] - .as_str() - .expect("registered daemon stop response text"), - ) - .expect("registered daemon stop payload"); - assert_eq!( - stop_payload["status"], "committed", - "registered daemon stop ingest did not commit: {stop_response}" - ); + // The project catch-up sweep races this pass for the rollout just written. + // Admission is the durable commit. A sweep that admits it first leaves the + // hook with nothing new to persist and reports `exact_duplicate`. Both + // terminals prove the transcript is durable; `accepted_for_replay` proves + // neither. A deferred or still-warming pass is the same typed progress the + // Cursor ingest above rides out. + let stop_deadline = std::time::Instant::now() + Duration::from_secs(60); + loop { + let stop_output = common::tracedecay_command_with_home(environment.home()) + .args([ + "tool", + "--project", + project_arg.as_str(), + "tracedecay_hook_runtime", + "--args", + stop_args.as_str(), + "--json", + ]) + .current_dir(&project) + .output() + .expect("invoke registered daemon stop path"); + if stop_output.status.success() { + let stop_response: Value = serde_json::from_slice(&stop_output.stdout) + .expect("registered daemon stop response"); + let stop_payload: Value = serde_json::from_str( + stop_response["content"][0]["text"] + .as_str() + .expect("registered daemon stop response text"), + ) + .expect("registered daemon stop payload"); + if stop_payload["completed"] != false { + assert!( + matches!( + stop_payload["status"].as_str(), + Some("committed" | "exact_duplicate") + ), + "registered daemon stop ingest proved neither a commit nor a duplicate: {stop_response}\ndaemon log:\n{}", + std::fs::read_to_string(&daemon_log) + .expect("read isolated advisory daemon log"), + ); + break; + } + assert_eq!( + stop_payload["admission"]["retryable"], true, + "incomplete stop ingest must carry a retryable admission: {stop_response}" + ); + } else { + let stderr = String::from_utf8_lossy(&stop_output.stderr).into_owned(); + assert!( + stderr.contains("is warming in the background"), + "registered daemon stop ingest failed\nstdout:\n{}\nstderr:\n{stderr}\ndaemon log:\n{}", + String::from_utf8_lossy(&stop_output.stdout), + std::fs::read_to_string(&daemon_log).expect("read isolated advisory daemon log"), + ); + } + assert!( + std::time::Instant::now() < stop_deadline, + "registered daemon stop ingest did not complete before its deadline\nstdout:\n{}\nstderr:\n{}\ndaemon log:\n{}", + String::from_utf8_lossy(&stop_output.stdout), + String::from_utf8_lossy(&stop_output.stderr), + std::fs::read_to_string(&daemon_log).expect("read isolated advisory daemon log"), + ); + tokio::time::sleep(Duration::from_millis(250)).await; + } let advisory_args = json!({ // Serialized as a file URL rather than concatenated: a Windows native diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/main.rs b/crates/tracedecay/tests/runtime_acceptance_suite/main.rs index d41e6cde75..7e9e00fe22 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/main.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/main.rs @@ -19,6 +19,10 @@ mod host_event_fixture_test; mod lifecycle_production_authority_test; mod private_route_restart_acceptance; mod runtime_surface_acceptance; +// Not implied by `test-transport`. An unconditional evaluator dependency +// unifies `tracedecay-query/search-eval` into every test target of this +// package, so the transport suites compile the eval-only lexical projection. +#[cfg(feature = "search-eval")] #[allow(clippy::option_env_unwrap)] mod search_eval_cli_test; #[cfg(unix)] diff --git a/crates/tracedecay/tests/session_suite/observation_projection/failure_audit.rs b/crates/tracedecay/tests/session_suite/observation_projection/failure_audit.rs index ea006aff50..c4b62d5177 100644 --- a/crates/tracedecay/tests/session_suite/observation_projection/failure_audit.rs +++ b/crates/tracedecay/tests/session_suite/observation_projection/failure_audit.rs @@ -767,15 +767,15 @@ async fn authority_reopen_accepts_historical_generation_after_supersession() { ); } -/// A projected message row is derived state, not authority: the immutable -/// observation plus its uniquely owned current provenance re-derive it exactly. -/// Since #1775 (`c55058a3ac`) the reopen audit therefore repairs a diverged -/// output row through the released-rendering convergence ledger instead of -/// degrading the profile forever. Provenance identity, digests that match -/// neither the current nor the stored output, foreign ownership, and -/// conflicting session fields remain hard failures. +/// A projected message body that matches neither this binary's rendering nor +/// the rendering a shipped release wrote is tamper. Profile reopen must name +/// that disagreement and leave the row untouched. An interrupted write whose +/// row is still the shipped rendering is a different admission and is not this +/// case. Provenance identity, digests that match neither the current nor the +/// stored output, foreign ownership, missing rows, and conflicting session +/// fields remain hard failures as well. #[tokio::test] -async fn projected_message_update_is_repaired_on_reopen() { +async fn projected_message_update_invalidates_audit_and_fails_reopen() { let tmp = audited_projection_fixture("session-audit-update", "message-audit-update").await; let runtime = profile_runtime(&tmp).await; let database_path = runtime @@ -793,19 +793,25 @@ async fn projected_message_update_is_repaired_on_reopen() { .unwrap(); drop(raw_conn); - let reopened = HostAdmissionTestRuntimeV1::profile(tmp.path().join(".tracedecay")) - .await - .expect("a diverged output row must be repaired, not refused"); - drop(reopened); + let Err(error) = HostAdmissionTestRuntimeV1::profile(tmp.path().join(".tracedecay")).await + else { + panic!("a tampered projected message must fail profile reopen"); + }; + let message = error.to_string(); + assert!( + message.contains("projection output rows disagree with deterministic output"), + "{message}" + ); assert!( - projected_message_texts(&tmp).await[0].contains("audited projection body"), - "reopen accepted the tampered body instead of re-projecting it" + projected_message_texts(&tmp).await[0].contains("tampered projection body"), + "profile reopen rewrote the tampered body instead of refusing it" ); } -/// The repair above covers a diverged row, never a vanished one: nothing in the -/// convergence ledger inserts a missing message row, so a store whose projected -/// output disappeared still has to be named rather than silently admitted. +/// A vanished projected output is the same hard failure: the convergence +/// ledger rewrites a shipped rendering it can see, and never inserts a missing +/// message row, so a store whose projected output disappeared still has to be +/// named rather than silently admitted. #[tokio::test] async fn projected_message_delete_invalidates_audit_and_fails_reopen() { let tmp = audited_projection_fixture("session-audit-delete", "message-audit-delete").await; diff --git a/crates/tracedecay/tests/session_suite/observation_store/mod.rs b/crates/tracedecay/tests/session_suite/observation_store/mod.rs index eeeb421313..47f430a1d5 100644 --- a/crates/tracedecay/tests/session_suite/observation_store/mod.rs +++ b/crates/tracedecay/tests/session_suite/observation_store/mod.rs @@ -1480,12 +1480,16 @@ async fn cursor_only_progress_persists_non_payload_receipt_and_retries_idempoten /// finds the durable cursor already at its `next_cursor`: the coverage it /// wanted to record is applied, so the replay is a duplicate, not a /// collision that blocks ingest (#1842). The committed reason stays. -async fn cursor_only_retry_with_same_cursor_and_different_reason_is_a_duplicate() { +async fn cursor_already_owned_keeps_the_first_reason_for_a_later_owner() { let tmp = TempDir::new().unwrap(); let runtime = profile_runtime(&tmp).await; let store = runtime .observation_store(HostAdmissionScope::Profile) .unwrap(); + let database_path = runtime + .database_path(HostAdmissionScope::Profile) + .unwrap() + .to_path_buf(); store .advance_source_cursor(cursor_advance( @@ -1513,10 +1517,24 @@ async fn cursor_only_retry_with_same_cursor_and_different_reason_is_a_duplicate( store.get_source_cursor(&source(), &scope()).await.unwrap(), Some(cursor(10)) ); + let conn = rusqlite::Connection::open(&database_path).unwrap(); + let reason: String = conn + .query_row("SELECT reason FROM source_cursor_advances", (), |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(reason, "blank_frame"); + assert_eq!( + conn.query_row("SELECT COUNT(*) FROM source_cursor_advances", (), |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 1 + ); } #[tokio::test] -async fn cursor_only_retry_rejects_same_cursor_with_different_coverage() { +async fn cursor_already_past_a_narrower_range_keeps_the_owned_frontier() { let tmp = TempDir::new().unwrap(); let runtime = profile_runtime(&tmp).await; let store = runtime @@ -1533,7 +1551,7 @@ async fn cursor_only_retry_rejects_same_cursor_with_different_coverage() { .await .unwrap(); - assert!(matches!( + assert_eq!( store .advance_source_cursor(cursor_advance( Some(cursor(5)), @@ -1541,9 +1559,10 @@ async fn cursor_only_retry_rejects_same_cursor_with_different_coverage() { 10, NonDurableFrameReason::BlankFrame, )) - .await, - Err(ObservationStoreError::CursorAdvanceCollision) - )); + .await + .unwrap(), + CursorAdvanceOutcome::ExactDuplicate + ); assert_eq!( store.get_source_cursor(&source(), &scope()).await.unwrap(), Some(cursor(10)) diff --git a/crates/tracedecay/tests/transport_acceptance_suite/daemon_fault_harness_test.rs b/crates/tracedecay/tests/transport_acceptance_suite/daemon_fault_harness_test.rs index dcf97a9b1d..57f416fee5 100644 --- a/crates/tracedecay/tests/transport_acceptance_suite/daemon_fault_harness_test.rs +++ b/crates/tracedecay/tests/transport_acceptance_suite/daemon_fault_harness_test.rs @@ -321,6 +321,75 @@ fn configured_daemon_can_be_killed_and_reaped() { } } +/// A listen descriptor inherited across `fork` must not outlive its owner. +/// +/// `branch_search_serves_a_committed_generation_behind_dirty_worktree_state` +/// drops the init daemon and immediately spawns the next one. Killing only the +/// leader leaves the inherited listener accepting on the same path, so the +/// next spawn reports a live daemon. The owner is the group leader; retiring +/// its socket with it makes that path refuse the moment the owner is reaped. +#[cfg(unix)] +#[test] +fn reaped_owner_releases_an_inherited_listen_socket() { + use std::os::unix::process::CommandExt; + use std::process::Stdio; + use std::time::{Duration, Instant}; + + let home = tempdir_or_panic(); + let socket_path = home.path().join("inherited-listen.sock"); + let script = r#" +import os, socket, sys, time +sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) +sock.bind(sys.argv[1]) +sock.listen(1) +if os.fork() == 0: + time.sleep(60) +else: + time.sleep(60) +"#; + let mut command = std::process::Command::new("python3"); + command + .arg("-c") + .arg(script) + .arg(&socket_path) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .process_group(0); + let child = command.spawn().expect("python listener should start"); + let mut owner = common::TestChildProcess::new(child); + owner.release_socket_on_stop(socket_path.clone()); + + let deadline = Instant::now() + Duration::from_secs(5); + loop { + if std::os::unix::net::UnixStream::connect(&socket_path).is_ok() { + break; + } + if let Some(status) = owner + .try_wait() + .expect("listener status should be readable") + { + panic!("listener exited before accepting: {status}"); + } + assert!( + Instant::now() < deadline, + "inherited listener never accepted on {}", + socket_path.display() + ); + std::thread::sleep(Duration::from_millis(20)); + } + + drop(owner); + assert!( + std::os::unix::net::UnixStream::connect(&socket_path).is_err(), + "an inherited listen descriptor must not keep the owner's path accepting" + ); + assert!( + !socket_path.exists(), + "reaping the owner must unlink the socket it bound" + ); +} + #[cfg(all(unix, tracedecay_observation_fault_harness, feature = "test-transport"))] async fn assert_daemon_crash_stage( barrier_stage: &str, diff --git a/scripts/lint-ci-commits.sh b/scripts/lint-ci-commits.sh new file mode 100755 index 0000000000..64a149863f --- /dev/null +++ b/scripts/lint-ci-commits.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Lint the commits a CI event is admitting. +# +# A master push lints github.event.before..HEAD, which is how an integration +# merge is judged after it lands. workflow_dispatch is the admission path for +# that integration branch, so it lints the same not-yet-on-the-default-branch +# range. Already published history is not rejudged: a dispatch of the default +# branch has an empty range. +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +project_root=$(cd "${script_dir}/.." && pwd) +repository=${REPOSITORY:-$project_root} +event=${EVENT_NAME:-} +head=${HEAD_SHA:-} +zero_sha=0000000000000000000000000000000000000000 + +if [[ -z $event || -z $head ]]; then + echo "usage: EVENT_NAME= HEAD_SHA= [BEFORE_SHA=] [DEFAULT_BRANCH=] [REPOSITORY=] $0" >&2 + exit 2 +fi + +lint_range() { + local base=$1 + node "${script_dir}/lint-commit-range.mjs" --repository "$repository" "$base" "$head" +} + +lint_root_commit() { + git -C "$repository" show --no-patch --format=%B "$head" | ( + cd "$project_root" + npm run --silent lint:commit -- + ) +} + +resolve_default_branch() { + local branch=$1 + local remote_ref="refs/remotes/origin/${branch}" + if git -C "$repository" remote get-url origin >/dev/null 2>&1; then + git -C "$repository" fetch --no-tags --quiet origin \ + "+refs/heads/${branch}:${remote_ref}" >/dev/null + fi + if git -C "$repository" rev-parse --verify --quiet "$remote_ref" >/dev/null; then + echo "$remote_ref" + return 0 + fi + if git -C "$repository" rev-parse --verify --quiet "refs/heads/${branch}" >/dev/null; then + echo "refs/heads/${branch}" + return 0 + fi + echo "commit lint: default branch ${branch} is not available" >&2 + return 1 +} + +case "$event" in + push) + before=${BEFORE_SHA:-} + if [[ -z $before ]]; then + echo "commit lint: push requires BEFORE_SHA" >&2 + exit 2 + fi + if [[ $before == "$zero_sha" ]]; then + if git -C "$repository" rev-parse --verify --quiet "${head}^" >/dev/null; then + lint_range "${head}^" + else + lint_root_commit + fi + else + lint_range "$before" + fi + ;; + workflow_dispatch) + default_branch=${DEFAULT_BRANCH:-} + if [[ -z $default_branch ]]; then + echo "commit lint: workflow_dispatch requires DEFAULT_BRANCH" >&2 + exit 2 + fi + upstream=$(resolve_default_branch "$default_branch") + base=$(git -C "$repository" merge-base "$head" "$upstream") + lint_range "$base" + ;; + *) + echo "commit lint: unsupported event ${event}" >&2 + exit 2 + ;; +esac diff --git a/scripts/test-lint-commit-range.py b/scripts/test-lint-commit-range.py index aac5532fd1..f58dd0f965 100755 --- a/scripts/test-lint-commit-range.py +++ b/scripts/test-lint-commit-range.py @@ -14,6 +14,21 @@ REPOSITORY_ROOT = Path(__file__).resolve().parent.parent LINT_RANGE = REPOSITORY_ROOT / "scripts" / "lint-commit-range.mjs" +LINT_CI = REPOSITORY_ROOT / "scripts" / "lint-ci-commits.sh" + +# Subjects from the integration fold that dispatch admitted and the following +# master push rejected. Each matches the typed-header grammar and fails only +# because the header is longer than the configured maximum. +BATCH_FOLLOWUP_SUBJECTS = ( + "fix(pr-1633): warm the diagnose fixture through the shared support helper", + "fix(pr-1740): resolve git through common::git_program and sort the mod line", + "fix(pr-1617): share the exact-arguments dispatch instead of widening CaptureTransport", +) +HYGIENIC_FOLLOWUP_MESSAGES = ( + "fix(pr-1633): warm the diagnose fixture through shared support\n\nhelper", + "fix(pr-1740): resolve git through common::git_program and sort mods\n\nthe mod line", + "fix(pr-1617): share exact-argument dispatch without widening transport\n\ninstead of widening CaptureTransport", +) def run( @@ -155,6 +170,134 @@ def test_node_startup_count_is_constant_for_a_large_range(self) -> None: f"elapsed_ms={elapsed_ms}" ) + def lint_ci( + self, + *, + event: str, + head: str, + before: str | None = None, + default_branch: str | None = None, + ) -> subprocess.CompletedProcess[str]: + environment = os.environ.copy() + environment.update( + { + "EVENT_NAME": event, + "HEAD_SHA": head, + "REPOSITORY": str(self.root), + } + ) + if before is not None: + environment["BEFORE_SHA"] = before + if default_branch is not None: + environment["DEFAULT_BRANCH"] = default_branch + return run( + ["bash", str(LINT_CI)], + cwd=self.root, + env=environment, + check=False, + ) + + def test_dispatch_rejects_batch_followup_headers_over_the_maximum(self) -> None: + base = self.commit("chore(test): establish fixture base") + master = self.commit("fix(test): keep the default branch valid", base) + run(["git", "branch", "master", master], cwd=self.root) + head = master + followups = [] + for subject in BATCH_FOLLOWUP_SUBJECTS: + self.assertGreater(len(subject), 72) + head = self.commit(subject, head) + followups.append(head) + + result = self.lint_ci( + event="workflow_dispatch", + head=head, + default_branch="master", + ) + output = result.stdout + result.stderr + + self.assertNotEqual(result.returncode, 0, output) + for sha, subject in zip(followups, BATCH_FOLLOWUP_SUBJECTS, strict=True): + self.assertIn(sha, output) + self.assertIn(subject, output) + self.assertIn("header-max-length", output) + self.assertNotIn(master, output) + + def test_dispatch_accepts_the_same_followups_once_the_header_fits(self) -> None: + base = self.commit("chore(test): establish fixture base") + master = self.commit("fix(test): keep the default branch valid", base) + run(["git", "branch", "master", master], cwd=self.root) + head = master + for message in HYGIENIC_FOLLOWUP_MESSAGES: + header = message.split("\n", 1)[0] + self.assertLessEqual(len(header), 72) + self.assertTrue(header.startswith("fix(pr-")) + head = self.commit(message, head) + + result = self.lint_ci( + event="workflow_dispatch", + head=head, + default_branch="master", + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_dispatch_of_the_default_branch_does_not_rejudge_published_history(self) -> None: + base = self.commit("chore(test): establish fixture base") + published = self.commit(BATCH_FOLLOWUP_SUBJECTS[0], base) + master = self.commit("fix(test): keep the default branch valid", published) + run(["git", "branch", "master", master], cwd=self.root) + + result = self.lint_ci( + event="workflow_dispatch", + head=master, + default_branch="master", + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn(published, result.stdout + result.stderr) + + def test_push_still_lints_the_before_sha_range(self) -> None: + base = self.commit("chore(test): establish fixture base") + head = self.commit(BATCH_FOLLOWUP_SUBJECTS[2], base) + + result = self.lint_ci(event="push", head=head, before=base) + output = result.stdout + result.stderr + + self.assertNotEqual(result.returncode, 0, output) + self.assertIn(head, output) + self.assertIn("header-max-length", output) + + def test_push_of_a_root_commit_lints_that_message(self) -> None: + valid = self.commit("chore(test): establish fixture base") + invalid = self.commit("not a conventional header") + + valid_result = self.lint_ci( + event="push", + head=valid, + before="0000000000000000000000000000000000000000", + ) + invalid_result = self.lint_ci( + event="push", + head=invalid, + before="0000000000000000000000000000000000000000", + ) + + self.assertEqual( + valid_result.returncode, + 0, + valid_result.stdout + valid_result.stderr, + ) + self.assertNotEqual(invalid_result.returncode, 0) + self.assertIn("type-empty", invalid_result.stdout + invalid_result.stderr) + + def test_unsupported_event_is_rejected(self) -> None: + head = self.commit("chore(test): establish fixture base") + + result = self.lint_ci(event="schedule", head=head) + + self.assertEqual(result.returncode, 2) + self.assertIn("unsupported event", result.stderr) + if __name__ == "__main__": unittest.main() diff --git a/tests/tool_sweep_suite/runner.py b/tests/tool_sweep_suite/runner.py index ec1b773f85..bb811bae15 100644 --- a/tests/tool_sweep_suite/runner.py +++ b/tests/tool_sweep_suite/runner.py @@ -716,8 +716,9 @@ def _mounting_producer_call( f"{tool} producer omitted the enabled _meta.duration_us receipt" ) return response + # Must match tracedecay_contracts::RUNTIME_MOUNTING_REASON_CODE. if ( - row["problem_code"] != "application.surface.unavailable" + row["problem_code"] != "application.runtime.mounting" or time.monotonic() >= ends_at ): raise SweepError(