diff --git a/crates/tracedecay-code-extraction/src/clone_body.rs b/crates/tracedecay-code-extraction/src/clone_body.rs index 29bb91be05..db94417b2e 100644 --- a/crates/tracedecay-code-extraction/src/clone_body.rs +++ b/crates/tracedecay-code-extraction/src/clone_body.rs @@ -1,7 +1,9 @@ use std::collections::HashMap; +use std::fmt; use std::sync::Arc; -use serde::{Deserialize, Serialize}; +use serde::de::{Error as _, MapAccess, Visitor}; +use serde::{Deserialize, Deserializer, Serialize}; use tracedecay_domain::{NodeKind, SourceSpan}; use tree_sitter::{Node as TreeSitterNode, Point, Tree, TreeCursor}; @@ -13,14 +15,163 @@ pub const CONSERVATIVE_CLONE_NORMALIZATION_REVISION_V1: u16 = 1; pub const RENAME_CLONE_NORMALIZATION_REVISION_V1: u16 = 1; pub const MIN_AUTOMATIC_CLONE_BODY_TOKENS_V1: u32 = 30; -#[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq, Hash)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +/// Clone-body token streams are the largest repeated record in a sealed +/// generation: a mid-size repository carries millions of these objects, and +/// every generation publish, restore, and clone-census read decodes all of +/// them. +/// +/// `Serialize` stays derived so the wire form remains serde's internally +/// tagged `{"kind":…,"syntax_kind":…,"text":…}` object, byte for byte. +/// `Deserialize` is written by hand because serde's derive for an internally +/// tagged enum buffers every object into `serde::__private::de::Content` — one +/// heap map plus owned key/value pairs per token — before it can dispatch on +/// the tag. The hand-written visitor reads the same object in one pass with no +/// intermediate buffer, and keeps the derive's refusals: an unknown or +/// duplicated member, a missing `kind`, an unknown tag, and a member that does +/// not belong to the tagged variant are all still errors. +#[derive(Clone, Debug, Serialize, Eq, PartialEq, Hash)] +#[serde(tag = "kind", rename_all = "snake_case")] pub enum ConservativeCloneTokenV1 { StructureStart { syntax_kind: String }, Syntax { syntax_kind: String, text: String }, StructureEnd { syntax_kind: String }, } +const CLONE_TOKEN_MEMBERS_V1: &[&str] = &["kind", "syntax_kind", "text"]; +const CLONE_TOKEN_STRUCTURE_MEMBERS_V1: &[&str] = &["kind", "syntax_kind"]; +const CLONE_TOKEN_TAGS_V1: &[&str] = &["structure_start", "syntax", "structure_end"]; + +#[derive(Clone, Copy)] +enum CloneTokenMemberV1 { + Kind, + SyntaxKind, + Text, +} + +#[derive(Clone, Copy)] +enum CloneTokenTagV1 { + StructureStart, + Syntax, + StructureEnd, +} + +impl<'de> Deserialize<'de> for CloneTokenMemberV1 { + fn deserialize>(deserializer: D) -> Result { + struct MemberVisitor; + + impl Visitor<'_> for MemberVisitor { + type Value = CloneTokenMemberV1; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a clone-body token member") + } + + fn visit_str(self, value: &str) -> Result { + match value { + "kind" => Ok(CloneTokenMemberV1::Kind), + "syntax_kind" => Ok(CloneTokenMemberV1::SyntaxKind), + "text" => Ok(CloneTokenMemberV1::Text), + other => Err(E::unknown_field(other, CLONE_TOKEN_MEMBERS_V1)), + } + } + } + + deserializer.deserialize_identifier(MemberVisitor) + } +} + +impl<'de> Deserialize<'de> for CloneTokenTagV1 { + fn deserialize>(deserializer: D) -> Result { + struct TagVisitor; + + impl Visitor<'_> for TagVisitor { + type Value = CloneTokenTagV1; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a clone-body token kind") + } + + fn visit_str(self, value: &str) -> Result { + match value { + "structure_start" => Ok(CloneTokenTagV1::StructureStart), + "syntax" => Ok(CloneTokenTagV1::Syntax), + "structure_end" => Ok(CloneTokenTagV1::StructureEnd), + other => Err(E::unknown_variant(other, CLONE_TOKEN_TAGS_V1)), + } + } + } + + deserializer.deserialize_str(TagVisitor) + } +} + +impl<'de> Deserialize<'de> for ConservativeCloneTokenV1 { + fn deserialize>(deserializer: D) -> Result { + struct TokenVisitor; + + impl<'de> Visitor<'de> for TokenVisitor { + type Value = ConservativeCloneTokenV1; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a clone-body token object") + } + + fn visit_map>(self, mut map: A) -> Result { + let mut tag = None; + let mut syntax_kind = None; + let mut text = None; + while let Some(member) = map.next_key::()? { + match member { + CloneTokenMemberV1::Kind => { + if tag.is_some() { + return Err(A::Error::duplicate_field("kind")); + } + tag = Some(map.next_value::()?); + } + CloneTokenMemberV1::SyntaxKind => { + if syntax_kind.is_some() { + return Err(A::Error::duplicate_field("syntax_kind")); + } + syntax_kind = Some(map.next_value::()?); + } + CloneTokenMemberV1::Text => { + if text.is_some() { + return Err(A::Error::duplicate_field("text")); + } + text = Some(map.next_value::()?); + } + } + } + let tag = tag.ok_or_else(|| A::Error::missing_field("kind"))?; + let syntax_kind = + syntax_kind.ok_or_else(|| A::Error::missing_field("syntax_kind"))?; + match tag { + CloneTokenTagV1::Syntax => Ok(ConservativeCloneTokenV1::Syntax { + syntax_kind, + text: text.ok_or_else(|| A::Error::missing_field("text"))?, + }), + CloneTokenTagV1::StructureStart | CloneTokenTagV1::StructureEnd + if text.is_some() => + { + Err(A::Error::unknown_field( + "text", + CLONE_TOKEN_STRUCTURE_MEMBERS_V1, + )) + } + CloneTokenTagV1::StructureStart => { + Ok(ConservativeCloneTokenV1::StructureStart { syntax_kind }) + } + CloneTokenTagV1::StructureEnd => { + Ok(ConservativeCloneTokenV1::StructureEnd { syntax_kind }) + } + } + } + } + + deserializer.deserialize_map(TokenVisitor) + } +} + #[derive(Clone, Copy, Debug, Serialize, Deserialize, Eq, PartialEq, Ord, PartialOrd, Hash)] #[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)] pub enum CloneBodyEligibilityV1 { diff --git a/crates/tracedecay-code-extraction/tests/main/clone_body_tokens.rs b/crates/tracedecay-code-extraction/tests/main/clone_body_tokens.rs index 081da3429f..2b747b9b12 100644 --- a/crates/tracedecay-code-extraction/tests/main/clone_body_tokens.rs +++ b/crates/tracedecay-code-extraction/tests/main/clone_body_tokens.rs @@ -253,3 +253,97 @@ fn clone_bodies_bind_to_method_and_stable_arrow_occurrences() { assert!(!body.body_span.is_empty()); } } + +/// The sealed generation stores clone-body token streams as internally tagged +/// objects, so the hand-written decoder has to reproduce the derived wire form +/// byte for byte and refuse everything the derive refused. +#[test] +fn clone_token_wire_form_round_trips_in_any_member_order() { + let stream = tokens( + &RustExtractor, + "src/lib.rs", + "fn publish(input: &str) -> bool { validate(parse(input), \"read\") }", + ); + let encoded = serde_json::to_string(&stream).expect("encode token stream"); + assert!( + encoded.starts_with(r#"[{"kind":"structure_start","syntax_kind":"#), + "clone-body tokens must keep the internally tagged wire form: {encoded}" + ); + assert!( + encoded.contains(r#"{"kind":"syntax","syntax_kind":"identifier","text":"validate"}"#), + "clone-body syntax tokens must keep tag-then-field member order: {encoded}" + ); + assert_eq!( + serde_json::from_str::>(&encoded).expect("decode"), + stream + ); + + // Member order is a serializer detail, never a decode requirement. + assert_eq!( + serde_json::from_str::( + r#"{"text":"validate","syntax_kind":"identifier","kind":"syntax"}"# + ) + .expect("decode reordered members"), + ConservativeCloneTokenV1::Syntax { + syntax_kind: "identifier".to_owned(), + text: "validate".to_owned(), + } + ); + assert_eq!( + serde_json::from_str::( + r#"{"syntax_kind":"block","kind":"structure_end"}"# + ) + .expect("decode reordered structure members"), + ConservativeCloneTokenV1::StructureEnd { + syntax_kind: "block".to_owned(), + } + ); +} + +#[test] +fn clone_token_decode_refuses_malformed_wire_objects() { + for (wire, expected) in [ + ( + r#"{"kind":"syntax","syntax_kind":"identifier","text":"a","extra":1}"#, + "unknown field `extra`", + ), + ( + r#"{"kind":"structure_start","syntax_kind":"block","text":"{"}"#, + "unknown field `text`", + ), + ( + r#"{"kind":"syntax","syntax_kind":"identifier"}"#, + "missing field `text`", + ), + (r#"{"syntax_kind":"block"}"#, "missing field `kind`"), + ( + r#"{"kind":"structure_start"}"#, + "missing field `syntax_kind`", + ), + ( + r#"{"kind":"structure_middle","syntax_kind":"block"}"#, + "unknown variant `structure_middle`", + ), + ( + r#"{"kind":"syntax","kind":"syntax","syntax_kind":"a","text":"b"}"#, + "duplicate field `kind`", + ), + ( + r#"{"kind":"syntax","syntax_kind":"a","syntax_kind":"a","text":"b"}"#, + "duplicate field `syntax_kind`", + ), + ( + r#"{"kind":"syntax","syntax_kind":"a","text":"b","text":"b"}"#, + "duplicate field `text`", + ), + (r#"["syntax","identifier","a"]"#, "invalid type"), + ] { + let error = serde_json::from_str::(wire) + .expect_err("malformed clone token must be refused") + .to_string(); + assert!( + error.contains(expected), + "decoding {wire} reported {error}, expected {expected}" + ); + } +} diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry.rs index a5392c83b1..c39518c389 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry.rs @@ -56,6 +56,8 @@ mod query_authority; mod reconcile_failure_isolation_tests; mod scope_identity; #[cfg(test)] +mod seat_swap_tests; +#[cfg(test)] mod serving_readiness_tests; mod serving_reads; @@ -352,6 +354,42 @@ impl ServingSwapOutcomeV1 { } } +/// The prepared generation id after a graph-activation failure. +/// +/// Retryable activation used to replace the prepared triple with +/// `Ok((Err, None, None))`, so the swap never ran and search kept the +/// predecessor for the whole backoff. Both retryable and terminal failures +/// now leave the sealed text generation in place; only graph readiness +/// retries or becomes unavailable. +pub(super) fn serving_generation_after_activation_failure<'a>( + prepared_generation: Option<&'a str>, + retryable: bool, + repeated_conflict: bool, +) -> Option<&'a str> { + if activation_failure_keeps_serving_candidate(retryable, repeated_conflict) { + prepared_generation + } else { + None + } +} + +fn activation_failure_keeps_serving_candidate(retryable: bool, repeated_conflict: bool) -> bool { + // `retryable && !repeated_conflict` used to wipe the candidate. Terminal + // failures already kept it. Both now keep it; the flags stay so a later + // change cannot drop only the retryable arm without this predicate. + let _ = (retryable, repeated_conflict); + true +} + +/// An unfinished text projection withholds the serving seat only when exact +/// or lexical owners are still missing. +/// +/// A clone-fingerprint successor keeps `text_projection_needs_work` after +/// those owners are ready. That is not `published_text_owner_unfinished`. +pub(super) fn text_projection_unfinished_withholds_seat(exact_and_lexical_ready: bool) -> bool { + !exact_and_lexical_ready +} + #[cfg(any(test, feature = "test-helpers"))] struct ColdMountFinalCommitGateV1 { project_root: PathBuf, diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs index ac239ce49c..ed8f3e88f6 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/mount.rs @@ -1638,6 +1638,11 @@ impl CodeIndexSchedulerRegistryV1 { // A conflict verdict identical to the previous // attempt's for this same generation is deterministic // and falls through to the terminal arm instead. + // + // The prepared text candidate stays. Wiping it to + // `Ok((Err, None, None))` skipped the serving swap, + // so search kept the predecessor while graph backoff + // ran. if error.is_retryable_activation() && !repeated_conflict { last_seat_conflict = error .activation_conflict_context() @@ -1654,7 +1659,7 @@ impl CodeIndexSchedulerRegistryV1 { retry_delay_micros = retry_delay.as_micros() as u64, error = %error, "graph activation failed retryably; the sealed generation \ - stays unseated until the scheduled retry" + still seats and the next pass retries native graph" ); hotpath::gauge!("daemon.code_index.graph_seat.retry_total") .inc(1_u64); @@ -1668,7 +1673,12 @@ impl CodeIndexSchedulerRegistryV1 { // The scheduled retry is the seat attempt, so it // must not be turned away as already attempted. graph_seat_attempted = None; - result = Ok((Err(error), None, None)); + if !super::activation_failure_keeps_serving_candidate( + error.is_retryable_activation(), + repeated_conflict, + ) { + result = Ok((Err(error), None, None)); + } } else { next_seat_attempt_at = None; seat_retry_backoff = ACTIVATION_RETRY_BACKOFF_FLOOR; @@ -1699,6 +1709,18 @@ impl CodeIndexSchedulerRegistryV1 { // and serving-swap boundary. Graph work above ran only when // the outcome was ready. if let Some(outcome) = published_text_projection_outcome.take() { + // A clone-fingerprint successor is still `Unfinished` work + // after exact and lexical owners are ready. That must not + // clear the prepared generation the way a missing owner does. + let owners_ready = exact_and_lexical_ready_for_graph(graph_text.as_ref()); + let outcome = match outcome { + PublishedTextProjectionOutcomeV1::Unfinished + if !super::text_projection_unfinished_withholds_seat(owners_ready) => + { + PublishedTextProjectionOutcomeV1::Finished + } + other => other, + }; match outcome { PublishedTextProjectionOutcomeV1::Finished => { // The seat needs only the ready exact/lexical diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/seat_swap_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/seat_swap_tests.rs new file mode 100644 index 0000000000..150a741eac --- /dev/null +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/seat_swap_tests.rs @@ -0,0 +1,45 @@ +use crate::code_index_scheduler::CodeIndexSchedulerErrorV1; + +use super::ServingSwapOutcomeV1; + +/// A retryable native-graph failure must not drop the generation search will +/// serve. The swap installs that same id; graph activation retries beside it. +#[test] +fn retryable_activation_keeps_the_serving_generation_matched() { + let error = CodeIndexSchedulerErrorV1::GraphActivation( + "graph runtime unavailable during activation".to_owned(), + ); + assert!( + error.is_retryable_activation(), + "GraphActivation is the retryable class that used to erase the seat candidate" + ); + let prepared = Some("generation.head"); + let seated = super::serving_generation_after_activation_failure( + prepared, + error.is_retryable_activation(), + false, + ); + assert_eq!( + seated, prepared, + "retryable graph activation must leave the prepared generation on the seat" + ); + let outcome = ServingSwapOutcomeV1::decide(true, true, seated.is_some()); + assert!( + outcome.installs(), + "the serving swap still writes the slot when the candidate survives: {outcome:?}" + ); +} + +/// Clone-fingerprint backfill is still unfinished after exact and lexical +/// owners are ready. That successor is not `published_text_owner_unfinished`. +#[test] +fn unfinished_clone_fingerprint_successor_is_not_text_projection_unfinished() { + assert!( + !super::text_projection_unfinished_withholds_seat(true), + "ready exact and lexical owners must still seat while the clone successor runs" + ); + assert!( + super::text_projection_unfinished_withholds_seat(false), + "missing exact or lexical owners still withhold the seat" + ); +} diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_census.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_census.rs index 19cd11ce27..232a152b81 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_census.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_census.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use rusqlite::Connection; use tracedecay_code_index::clones::{ CloneBodyEligibilityV1, CloneBodyOccurrenceV1, CloneBodyPayloadV1, CloneBodyRenameStatusV1, @@ -24,15 +26,70 @@ pub struct CodeLexicalCloneIndexCensusV1 { pub rename_unsupported_bodies: u64, } +/// The rename coverages the per-occurrence counters distinguish. `Complete` +/// bumps no counter, so it is never retained and the census cannot hold a +/// coverage it would then have to drop. +#[derive(Clone, Copy)] +enum IncompleteRenameCoverageV1 { + Partial, + UnsupportedLanguage, +} + +/// Validate every stored clone payload once and report which of them lack +/// complete rename normalization. +/// +/// `clone_body_payloads` is keyed by payload digest, so one row backs every +/// occurrence that shares that body. Re-deriving a payload's four canonical +/// digests once per occurrence therefore repeated the same verification for +/// every duplicate: a generated 768-file corpus stores 98,304 occurrences over +/// 128 distinct payloads, and the census spent ~1.2 s of single-threaded +/// verification where 128 validations were the whole obligation. +/// +/// Only the non-`Complete` coverages are retained, because those are the only +/// ones the per-occurrence rename counters distinguish. +fn validate_stored_clone_payloads( + connection: &Connection, +) -> Result, CodeLexicalArtifactErrorV1> { + let mut incomplete_rename = HashMap::new(); + let mut statement = connection + .prepare("SELECT payload_digest, payload FROM clone_body_payloads ORDER BY payload_digest") + .map_err(sqlite_error)?; + let mut rows = statement.query([]).map_err(sqlite_error)?; + while let Some(row) = rows.next().map_err(sqlite_error)? { + let digest: String = row.get(0).map_err(sqlite_error)?; + let payload_bytes: Vec = row.get(1).map_err(sqlite_error)?; + let payload: CloneBodyPayloadV1 = serde_json::from_slice(&payload_bytes) + .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; + if payload.payload_digest.as_str() != digest || payload.validate().is_err() { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "clone census found a payload outside its stored digest".to_owned(), + )); + } + match payload.rename_coverage { + CloneBodyRenameStatusV1::Complete => {} + CloneBodyRenameStatusV1::Partial => { + incomplete_rename.insert(digest, IncompleteRenameCoverageV1::Partial); + } + CloneBodyRenameStatusV1::UnsupportedLanguage => { + incomplete_rename.insert(digest, IncompleteRenameCoverageV1::UnsupportedLanguage); + } + } + } + Ok(incomplete_rename) +} + pub(super) fn read_clone_index_census( connection: &Connection, has_fingerprints: bool, hot_posting_threshold: u64, ) -> Result { let mut census = CodeLexicalCloneIndexCensusV1::default(); + let incomplete_rename = validate_stored_clone_payloads(connection)?; + // The inner join proves every counted occurrence has its verified payload + // row; the occurrence total below proves none was dropped by it. let mut statement = connection .prepare( - "SELECT occurrence.occurrence, payload.payload + "SELECT occurrence.payload_digest, occurrence.occurrence FROM clone_occurrences AS occurrence JOIN clone_body_payloads AS payload ON payload.payload_digest = occurrence.payload_digest @@ -41,13 +98,11 @@ pub(super) fn read_clone_index_census( .map_err(sqlite_error)?; let mut rows = statement.query([]).map_err(sqlite_error)?; while let Some(row) = rows.next().map_err(sqlite_error)? { - let occurrence_bytes: Vec = row.get(0).map_err(sqlite_error)?; - let payload_bytes: Vec = row.get(1).map_err(sqlite_error)?; + let digest: String = row.get(0).map_err(sqlite_error)?; + let occurrence_bytes: Vec = row.get(1).map_err(sqlite_error)?; let occurrence: CloneBodyOccurrenceV1 = serde_json::from_slice(&occurrence_bytes) .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; - let payload: CloneBodyPayloadV1 = serde_json::from_slice(&payload_bytes) - .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; - if occurrence.payload_digest != payload.payload_digest || payload.validate().is_err() { + if occurrence.payload_digest.as_str() != digest { return Err(CodeLexicalArtifactErrorV1::Corrupt( "clone census found a payload outside its occurrence binding".to_owned(), )); @@ -56,13 +111,13 @@ pub(super) fn read_clone_index_census( match occurrence.eligibility { CloneBodyEligibilityV1::Eligible => { census.eligible_source_bodies = census.eligible_source_bodies.saturating_add(1); - match payload.rename_coverage { - CloneBodyRenameStatusV1::Complete => {} - CloneBodyRenameStatusV1::Partial => { + match incomplete_rename.get(&digest) { + None => {} + Some(IncompleteRenameCoverageV1::Partial) => { census.rename_partial_bodies = census.rename_partial_bodies.saturating_add(1); } - CloneBodyRenameStatusV1::UnsupportedLanguage => { + Some(IncompleteRenameCoverageV1::UnsupportedLanguage) => { census.rename_unsupported_bodies = census.rename_unsupported_bodies.saturating_add(1); } @@ -82,23 +137,43 @@ pub(super) fn read_clone_index_census( drop(rows); drop(statement); - let (unique_payloads, exact_postings, conservative, rename): (i64, i64, i64, i64) = connection + let (unique_payloads, exact_postings, conservative, rename, occurrences): ( + i64, + i64, + i64, + i64, + i64, + ) = connection .query_row( "SELECT (SELECT COUNT(*) FROM clone_body_payloads), (SELECT COUNT(*) FROM clone_exact_postings), (SELECT COUNT(*) FROM clone_exact_postings WHERE class = ?1), - (SELECT COUNT(*) FROM clone_exact_postings WHERE class = ?2)", + (SELECT COUNT(*) FROM clone_exact_postings WHERE class = ?2), + (SELECT COUNT(*) FROM clone_occurrences)", [ i64::from(CloneNormalizationClassV1::Conservative as u8), i64::from(CloneNormalizationClassV1::Rename as u8), ], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }, ) .map_err(sqlite_error)?; let count = |value: i64| -> Result { u64::try_from(value).map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string())) }; + if count(occurrences)? != census.source_bodies { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "clone census found an occurrence without its payload".to_owned(), + )); + } census.unique_payloads = count(unique_payloads)?; census.exact_postings = count(exact_postings)?; census.conservative_normalized_bodies = count(conservative)?;