From eeb1ec1d1c4f0c658292f5340660e1302f9ff481 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 18:55:48 +0000 Subject: [PATCH 1/3] fix: clear the deterministic master CI failures The npm policy checker still required a tag trigger after publication moved to a master dispatch. An expired source proof raced the reconcile worker. Clone warmup faults that are not contract violations were a permanent search failure while a generation was retiring. --- .../src/code_index_executor.rs | 16 ++++++++-------- .../src/code_index_scheduler/tests/reconcile.rs | 7 ++++++- .../tests/sdk_suite/production_daemon.rs | 8 +++++++- crates/tracedecay/src/daemon/http_application.rs | 5 ++++- scripts/check-sdk-publish-workflow.py | 13 ++++++++----- scripts/test-check-sdk-publish-workflow.py | 14 +++++++++++--- 6 files changed, 44 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_executor.rs b/crates/tracedecay-code-index-runtime/src/code_index_executor.rs index a9e79db843..e55b9aef86 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_executor.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_executor.rs @@ -1506,14 +1506,14 @@ where // state `Pending` reports above, so it keeps `Pending`'s // retryable verdict; `Internal` told callers never to retry a // window that resolves itself within one background pass. - Err(RetrievalPortError::AuthorityUnavailable(_)) => { + Err(RetrievalPortError::Contract(_)) => { + return unavailable(code_search::CodeIndexSearchUnavailableReasonV1::Internal); + } + Err(_) => { return unavailable( code_search::CodeIndexSearchUnavailableReasonV1::GenerationUnverified, ); } - Err(_) => { - return unavailable(code_search::CodeIndexSearchUnavailableReasonV1::Internal); - } } let owners = match generation.production_query_owners_with_budget( &code_index_scheduler::queries::maximum_retrieval_budget(), @@ -1712,14 +1712,14 @@ where // state `Pending` reports above, so it keeps `Pending`'s // retryable verdict; `Internal` told callers never to retry a // window that resolves itself within one background pass. - Err(RetrievalPortError::AuthorityUnavailable(_)) => { + Err(RetrievalPortError::Contract(_)) => { + return unavailable(code_search::CodeIndexSearchUnavailableReasonV1::Internal); + } + Err(_) => { return unavailable( code_search::CodeIndexSearchUnavailableReasonV1::GenerationUnverified, ); } - Err(_) => { - return unavailable(code_search::CodeIndexSearchUnavailableReasonV1::Internal); - } } let owners = match generation.production_query_owners_with_budget( &code_index_scheduler::queries::maximum_retrieval_budget(), diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs index ebe1870245..6e7373440b 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs @@ -2690,7 +2690,11 @@ async fn long_text_projection_renews_source_before_seating_and_noop_follow_up_se drain_clone_backfill(®istry, fixture.path()).await; // Exercise the ordinary expiry path too. The existing seat keeps its exact - // witness while the source-verification Noop renews the proof. + // witness while the source-verification Noop renews the proof. Hold the + // worker out until the decline is observed: a pass that wins the race + // renews the proof before this assert and the configured fail-on-flake + // gate treats that as a failure. + let admission = quiesced_background_reconcile_admission(®istry, fixture.path()).await; { let mut state = source_freshness .state @@ -2709,6 +2713,7 @@ async fn long_text_projection_renews_source_before_seating_and_noop_follow_up_se .is_none(), "the expired proof declines before the worker renews it" ); + drop(admission); assert_eq!( wait_until_serving_seat(®istry, fixture.path(), Duration::from_secs(10), || { registry.latest_complete_ready_decoded_for_root_scope(fixture.path(), &scope) diff --git a/crates/tracedecay-sdk/tests/sdk_suite/production_daemon.rs b/crates/tracedecay-sdk/tests/sdk_suite/production_daemon.rs index ec803de922..6e2ead1c76 100644 --- a/crates/tracedecay-sdk/tests/sdk_suite/production_daemon.rs +++ b/crates/tracedecay-sdk/tests/sdk_suite/production_daemon.rs @@ -303,7 +303,13 @@ fn enrolled_remote_client_rejects_an_untrusted_private_authority_and_isolates_en .json(&json!({"grant": grant, "admission": admission})) .send() .unwrap(); - assert_eq!(provisioned.status(), reqwest::StatusCode::NO_CONTENT); + let provision_status = provisioned.status(); + let provision_body = provisioned.text().unwrap_or_default(); + assert_eq!( + provision_status, + reqwest::StatusCode::NO_CONTENT, + "{provision_body}" + ); let request = enrollment_request(&grant); let untrusted_authority = EnrolledRemoteClient::new_with_root_certificate( diff --git a/crates/tracedecay/src/daemon/http_application.rs b/crates/tracedecay/src/daemon/http_application.rs index 73f602856a..543477933f 100644 --- a/crates/tracedecay/src/daemon/http_application.rs +++ b/crates/tracedecay/src/daemon/http_application.rs @@ -400,7 +400,10 @@ async fn provision_remote_node( .await { Ok(()) => StatusCode::NO_CONTENT.into_response(), - Err(_) => StatusCode::CONFLICT.into_response(), + Err(error) => { + tracing::error!(%error, "remote node provision failed"); + (StatusCode::CONFLICT, error.to_string()).into_response() + } } } diff --git a/scripts/check-sdk-publish-workflow.py b/scripts/check-sdk-publish-workflow.py index 1cfb252ced..337610e6f2 100644 --- a/scripts/check-sdk-publish-workflow.py +++ b/scripts/check-sdk-publish-workflow.py @@ -64,11 +64,14 @@ def assert_actions_pinned(name: str, job: dict[str, Any]) -> None: def assert_release_trigger(workflow: dict[str, Any]) -> None: triggers = workflow.get("on", workflow.get(True, {})) - if not isinstance(triggers, dict) or set(triggers) != {"release", "workflow_dispatch"}: - fail("npm publication must ride the GitHub Release trigger plus tag recovery only") - release = triggers.get("release") - if not isinstance(release, dict) or release.get("types") != ["published"]: - fail("the release trigger must fire on published releases only") + # A run whose ref is the release tag cannot restore the previous release's + # Actions cache. release-please dispatches this workflow on master after + # the immutable tag exists, so a `release` trigger is not an allowed path. + if not isinstance(triggers, dict) or set(triggers) != {"workflow_dispatch"}: + fail( + "npm publication must be dispatched on master after the immutable " + "tag exists; a tag-ref run cannot restore the release cache" + ) dispatch = triggers.get("workflow_dispatch") inputs = dispatch.get("inputs") if isinstance(dispatch, dict) else None if not isinstance(inputs, dict) or set(inputs) != {"release_tag"}: diff --git a/scripts/test-check-sdk-publish-workflow.py b/scripts/test-check-sdk-publish-workflow.py index 1b1a8e79a8..2d6c9e1e17 100644 --- a/scripts/test-check-sdk-publish-workflow.py +++ b/scripts/test-check-sdk-publish-workflow.py @@ -36,10 +36,18 @@ def assert_rejected(self, workflow: str) -> None: with self.assertRaises(SystemExit): self.checker.main() - def test_rejects_dropping_the_release_trigger(self) -> None: + def test_rejects_dropping_the_release_dispatch(self) -> None: mutated = self.workflow.replace( - "on:\n release:\n types: [published]\n workflow_dispatch:", - "on:\n workflow_dispatch:", + "on:\n" + " workflow_dispatch:\n" + " inputs:\n" + " release_tag:\n" + ' description: "Stable release tag to build or recover"\n' + " required: true\n" + " type: string\n", + "on:\n" + " push:\n" + " branches: [master]\n", 1, ) self.assert_rejected(mutated) From ab0bd0652d9eeb66d1091243ec7fd66ef1d3aa31 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 19:15:15 +0000 Subject: [PATCH 2/3] revert(code-index): keep the terminal clone-warmup verdict `advance_text_serving_for_request` calls `mark_text_serving_failed` for `CapabilityManifestRejected`, `GenerationMismatch`, `IncompatibleProjection` and `Contract` (serving.rs:2589). Mapping every non-`Contract` error to `GenerationUnverified` told callers to retry a projection that the same call had just marked permanently failed, so the three incompatibility variants became an unbounded retry on a terminal state. 024364a1f1 already maps the observed retiring-staging fault (`AuthorityUnavailable`) to `generation_unverified`, and cacf79a8d2 root-caused the remaining clone-lane refusal as the head-open `Contract` violation and fixed it at source, recording that a retryable verdict would only have hidden it. No evidence names a further variant, so keep master's mapping. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/code_index_executor.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_executor.rs b/crates/tracedecay-code-index-runtime/src/code_index_executor.rs index e55b9aef86..a9e79db843 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_executor.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_executor.rs @@ -1506,14 +1506,14 @@ where // state `Pending` reports above, so it keeps `Pending`'s // retryable verdict; `Internal` told callers never to retry a // window that resolves itself within one background pass. - Err(RetrievalPortError::Contract(_)) => { - return unavailable(code_search::CodeIndexSearchUnavailableReasonV1::Internal); - } - Err(_) => { + Err(RetrievalPortError::AuthorityUnavailable(_)) => { return unavailable( code_search::CodeIndexSearchUnavailableReasonV1::GenerationUnverified, ); } + Err(_) => { + return unavailable(code_search::CodeIndexSearchUnavailableReasonV1::Internal); + } } let owners = match generation.production_query_owners_with_budget( &code_index_scheduler::queries::maximum_retrieval_budget(), @@ -1712,14 +1712,14 @@ where // state `Pending` reports above, so it keeps `Pending`'s // retryable verdict; `Internal` told callers never to retry a // window that resolves itself within one background pass. - Err(RetrievalPortError::Contract(_)) => { - return unavailable(code_search::CodeIndexSearchUnavailableReasonV1::Internal); - } - Err(_) => { + Err(RetrievalPortError::AuthorityUnavailable(_)) => { return unavailable( code_search::CodeIndexSearchUnavailableReasonV1::GenerationUnverified, ); } + Err(_) => { + return unavailable(code_search::CodeIndexSearchUnavailableReasonV1::Internal); + } } let owners = match generation.production_query_owners_with_budget( &code_index_scheduler::queries::maximum_retrieval_budget(), From 656cca971c812a45fb8661f9849d4d03112f700c Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 19 Sep 2026 19:28:38 +0000 Subject: [PATCH 3/3] test(release): reject restoring the npm release trigger The policy now forbids the `release` trigger because a run on the release tag cannot restore the previous release's Actions cache (3ac7ae1640), but no test covered that rule: the suite only checked that the master dispatch is present. Re-adding `release: types: [published]` passed. Add the mirror mutation, and rename `assert_release_trigger` to `assert_master_dispatch_trigger` so the function no longer names the trigger it rejects. Verified by weakening the checker to accept `{release, workflow_dispatch}` again: the new test is the only one that fails. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/check-sdk-publish-workflow.py | 4 ++-- scripts/test-check-sdk-publish-workflow.py | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/check-sdk-publish-workflow.py b/scripts/check-sdk-publish-workflow.py index 337610e6f2..bce28e140a 100644 --- a/scripts/check-sdk-publish-workflow.py +++ b/scripts/check-sdk-publish-workflow.py @@ -62,7 +62,7 @@ def assert_actions_pinned(name: str, job: dict[str, Any]) -> None: fail(f"'{name}' uses unpinned action {uses!r}") -def assert_release_trigger(workflow: dict[str, Any]) -> None: +def assert_master_dispatch_trigger(workflow: dict[str, Any]) -> None: triggers = workflow.get("on", workflow.get(True, {})) # A run whose ref is the release tag cannot restore the previous release's # Actions cache. release-please dispatches this workflow on master after @@ -207,7 +207,7 @@ def main() -> None: ) workflow = yaml.safe_load(text) - assert_release_trigger(workflow) + assert_master_dispatch_trigger(workflow) if workflow.get("permissions") != {"contents": "read"}: fail("top-level permissions must grant contents: read only") diff --git a/scripts/test-check-sdk-publish-workflow.py b/scripts/test-check-sdk-publish-workflow.py index 2d6c9e1e17..74b129cab8 100644 --- a/scripts/test-check-sdk-publish-workflow.py +++ b/scripts/test-check-sdk-publish-workflow.py @@ -52,6 +52,14 @@ def test_rejects_dropping_the_release_dispatch(self) -> None: ) self.assert_rejected(mutated) + def test_rejects_restoring_the_release_trigger(self) -> None: + mutated = self.workflow.replace( + "on:\n workflow_dispatch:\n", + "on:\n release:\n types: [published]\n workflow_dispatch:\n", + 1, + ) + self.assert_rejected(mutated) + def test_rejects_sdk_dispatch_selector(self) -> None: mutated = self.workflow.replace( " workflow_dispatch:\n inputs:\n release_tag:",