From 56bf72c816008b6759e962bf479b5cf71506f796 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 20 Sep 2026 04:18:22 +0000 Subject: [PATCH 1/2] perf(code-index): sample git metadata through retained topology `GitMetadataFingerprintV1::capture` is the tier-1 staleness signal sampled on every query admission, and this module's own contract calls that cost fixed and cheap. Resolving the git-dir and common-dir through a fresh `gix::open` dominated it, while runtime-core already owns a revalidating topology memo that answers the same question. The memo is asked only for a checkout carrying `/.git`, which is both where an open at exactly this root resolves through and where a discovery started at this root stops, so it returns the same two paths. A bare repository's control directory, or a path that is not a checkout root, still opens directly, because discovery would walk past it to an ancestor whose git metadata does not describe this project. The fingerprint samples file metadata and contents, so its value and its persisted signature are unchanged. Measured by the ignored `measure_git_metadata_fingerprint_capture` test added here, one-ref fixture repository, 2000 warm iterations after a 100-iteration warmup: 127.9us per capture before, 27.7us after. Ports e4290b05e8 from #1580, where the same change measured 75.2us to 10.0us on different hardware. Co-Authored-By: Claude Fable 5.1 --- .../src/code_index_scheduler/identity.rs | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs index b10ffc58a8..1c57277969 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs @@ -269,7 +269,23 @@ fn refs_heads_signature(dir: &Path) -> Option { /// Resolve the git-dir (worktree-local) and common-dir (repository-shared) /// paths, falling back to `/.git` when gix cannot open the checkout so a /// non-repository path still yields a stable, if empty, fingerprint. +/// +/// These two paths are structural, but the fingerprint above is sampled on +/// every query admission, and re-deriving them through a fresh repository open +/// dominates its cost, against the retained topology this now asks first. A +/// checkout that carries `/.git` is one an open at exactly this root +/// resolves through, which is also where a discovery started at this root +/// stops, so the retained answer is the same answer. Anything else, a bare +/// repository's control directory or a path that is not a checkout root, still +/// opens directly, because discovery would walk past it to an ancestor whose +/// git metadata does not describe this project. fn git_metadata_dirs(project_root: &Path) -> (PathBuf, PathBuf) { + if project_root.join(".git").exists() + && let Ok(topology) = + tracedecay_runtime_core::git_repository::repository_topology(project_root) + { + return (topology.git_dir.clone(), topology.common_dir.clone()); + } if let Ok(repository) = gix::open(project_root) { let git_dir = repository.git_dir().to_path_buf(); let common_dir = { @@ -487,4 +503,26 @@ mod tests { "an in-place loose-ref rewrite must be detected by the refs signature" ); } + + /// Cost of one tier-1 staleness sample, the number this module's contract + /// calls fixed and cheap. + /// + /// Ignored because it reports a duration rather than asserting one; run it + /// with `--ignored --nocapture` to re-derive the figure in the module doc. + #[test] + #[ignore = "timing measurement, not a pass/fail contract"] + fn measure_git_metadata_fingerprint_capture() { + const ITERATIONS: u32 = 2000; + + let repo = init_repo(&[("src/lib.rs", "pub fn a() {}\n")]); + for _ in 0..100 { + std::hint::black_box(GitMetadataFingerprintV1::capture(repo.path())); + } + let started = std::time::Instant::now(); + for _ in 0..ITERATIONS { + std::hint::black_box(GitMetadataFingerprintV1::capture(repo.path())); + } + let per_capture = started.elapsed() / ITERATIONS; + println!("capture: {:.1}us", per_capture.as_secs_f64() * 1e6); + } } From c18b3e611da19db2ebf555eb2811cefcffff4a9b Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sun, 20 Sep 2026 04:18:26 +0000 Subject: [PATCH 2/2] perf(query): validate each stored clone payload once per census `clone_body_payloads` is keyed by payload digest, so one row backs every occurrence that shares that body. The census joined the two tables and re-derived a payload's four canonical digests once per occurrence, so a corpus of 128 distinct payloads verified those 128 payloads 98,304 times, inside a status read that holds the scheduler lock. Validate the payload table once, retain only the rename coverages the per-occurrence counters distinguish, and classify occurrences against that verified index. Both prior refusals are kept and two are added: a payload whose blob disagrees with its stored digest, and an occurrence whose payload row is absent. The inner join used to drop that second case from the totals without saying so, reporting a healthy census over fewer bodies than the artifact holds, which is worse than a refusal because nothing downstream can tell the count is short. The new `census_refuses_an_occurrence_whose_payload_row_is_absent` test pins it, and fails against the previous census with `source_bodies: 1` for two stored occurrences. Measured by the ignored `measure_read_clone_index_census` test added here, 98,304 occurrences over 128 payloads: 2.282s before, 0.310s after. Ports ea06d59df9 from #1577. Co-Authored-By: Claude Fable 5.1 --- .../projection/artifact/clone_census.rs | 267 +++++++++++++++++- 1 file changed, 254 insertions(+), 13 deletions(-) diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_census.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_census.rs index a6661692a7..967872c365 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_census.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_census.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use rusqlite::Connection; use tracedecay_code_index::clones::{ CloneBodyEligibilityV1, CloneBodyOccurrenceV1, CloneBodyPayloadV1, CloneBodyRenameStatusV1, @@ -25,15 +27,70 @@ pub struct CodeLexicalCloneIndexCensusV1 { pub rename_unsupported_bodies: u64, } +/// The rename coverages the per-occurrence counters distinguish. `Complete` +/// bumps no counter, so it is never retained and the census cannot hold a +/// coverage it would then have to drop. +#[derive(Clone, Copy)] +enum IncompleteRenameCoverageV1 { + Partial, + UnsupportedLanguage, +} + +/// Validate every stored clone payload once and report which of them lack +/// complete rename normalization. +/// +/// `clone_body_payloads` is keyed by payload digest, so one row backs every +/// occurrence that shares that body. Re-deriving a payload's four canonical +/// digests once per occurrence therefore repeated the same verification for +/// every duplicate: a generated 768-file corpus stores 98,304 occurrences over +/// 128 distinct payloads, and the census spent ~1.2 s of single-threaded +/// verification where 128 validations were the whole obligation. +/// +/// Only the non-`Complete` coverages are retained, because those are the only +/// ones the per-occurrence rename counters distinguish. +fn validate_stored_clone_payloads( + connection: &Connection, +) -> Result, CodeLexicalArtifactErrorV1> { + let mut incomplete_rename = HashMap::new(); + let mut statement = connection + .prepare("SELECT payload_digest, payload FROM clone_body_payloads ORDER BY payload_digest") + .map_err(sqlite_error)?; + let mut rows = statement.query([]).map_err(sqlite_error)?; + while let Some(row) = rows.next().map_err(sqlite_error)? { + let digest: String = row.get(0).map_err(sqlite_error)?; + let payload_bytes: Vec = row.get(1).map_err(sqlite_error)?; + let payload: CloneBodyPayloadV1 = serde_json::from_slice(&payload_bytes) + .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; + if payload.payload_digest.as_str() != digest || payload.validate().is_err() { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "clone census found a payload outside its stored digest".to_owned(), + )); + } + match payload.rename_coverage { + CloneBodyRenameStatusV1::Complete => {} + CloneBodyRenameStatusV1::Partial => { + incomplete_rename.insert(digest, IncompleteRenameCoverageV1::Partial); + } + CloneBodyRenameStatusV1::UnsupportedLanguage => { + incomplete_rename.insert(digest, IncompleteRenameCoverageV1::UnsupportedLanguage); + } + } + } + Ok(incomplete_rename) +} + pub(super) fn read_clone_index_census( connection: &Connection, has_fingerprints: bool, hot_posting_threshold: u64, ) -> Result { let mut census = CodeLexicalCloneIndexCensusV1::default(); + let incomplete_rename = validate_stored_clone_payloads(connection)?; + // The inner join proves every counted occurrence has its verified payload + // row; the occurrence total below proves none was dropped by it. let mut statement = connection .prepare( - "SELECT occurrence.occurrence, payload.payload + "SELECT occurrence.payload_digest, occurrence.occurrence FROM clone_occurrences AS occurrence JOIN clone_body_payloads AS payload ON payload.payload_digest = occurrence.payload_digest @@ -42,13 +99,11 @@ pub(super) fn read_clone_index_census( .map_err(sqlite_error)?; let mut rows = statement.query([]).map_err(sqlite_error)?; while let Some(row) = rows.next().map_err(sqlite_error)? { - let occurrence_bytes: Vec = row.get(0).map_err(sqlite_error)?; - let payload_bytes: Vec = row.get(1).map_err(sqlite_error)?; + let digest: String = row.get(0).map_err(sqlite_error)?; + let occurrence_bytes: Vec = row.get(1).map_err(sqlite_error)?; let occurrence: CloneBodyOccurrenceV1 = serde_json::from_slice(&occurrence_bytes) .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; - let payload: CloneBodyPayloadV1 = serde_json::from_slice(&payload_bytes) - .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; - if occurrence.payload_digest != payload.payload_digest || payload.validate().is_err() { + if occurrence.payload_digest.as_str() != digest { return Err(CodeLexicalArtifactErrorV1::Corrupt( "clone census found a payload outside its occurrence binding".to_owned(), )); @@ -57,13 +112,13 @@ pub(super) fn read_clone_index_census( match occurrence.eligibility { CloneBodyEligibilityV1::Eligible => { census.eligible_source_bodies = census.eligible_source_bodies.saturating_add(1); - match payload.rename_coverage { - CloneBodyRenameStatusV1::Complete => {} - CloneBodyRenameStatusV1::Partial => { + match incomplete_rename.get(&digest) { + None => {} + Some(IncompleteRenameCoverageV1::Partial) => { census.rename_partial_bodies = census.rename_partial_bodies.saturating_add(1); } - CloneBodyRenameStatusV1::UnsupportedLanguage => { + Some(IncompleteRenameCoverageV1::UnsupportedLanguage) => { census.rename_unsupported_bodies = census.rename_unsupported_bodies.saturating_add(1); } @@ -87,23 +142,43 @@ pub(super) fn read_clone_index_census( drop(rows); drop(statement); - let (unique_payloads, exact_postings, conservative, rename): (i64, i64, i64, i64) = connection + let (unique_payloads, exact_postings, conservative, rename, occurrences): ( + i64, + i64, + i64, + i64, + i64, + ) = connection .query_row( "SELECT (SELECT COUNT(*) FROM clone_body_payloads), (SELECT COUNT(*) FROM clone_exact_postings), (SELECT COUNT(*) FROM clone_exact_postings WHERE class = ?1), - (SELECT COUNT(*) FROM clone_exact_postings WHERE class = ?2)", + (SELECT COUNT(*) FROM clone_exact_postings WHERE class = ?2), + (SELECT COUNT(*) FROM clone_occurrences)", [ i64::from(CloneNormalizationClassV1::Conservative as u8), i64::from(CloneNormalizationClassV1::Rename as u8), ], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }, ) .map_err(sqlite_error)?; let count = |value: i64| -> Result { u64::try_from(value).map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string())) }; + if count(occurrences)? != census.source_bodies { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "clone census found an occurrence without its payload".to_owned(), + )); + } census.unique_payloads = count(unique_payloads)?; census.exact_postings = count(exact_postings)?; census.conservative_normalized_bodies = count(conservative)?; @@ -135,3 +210,169 @@ pub(super) fn read_clone_index_census( } Ok(census) } + +#[cfg(test)] +mod tests { + use super::*; + use rusqlite::params; + use std::sync::Arc; + use tracedecay_code_extraction::{ + CloneBodyTokenizationStatusV1, ConservativeCloneTokenV1, ExtractedCloneBodyV1, + }; + use tracedecay_domain::{ + CodeGenerationId, NodeKind, ProjectId, RepositoryId, SourceSpan, SymbolOccurrenceId, + }; + + /// The three tables the census reads. Triggers and the builder gate belong + /// to the write path, which no census read goes through. + fn census_schema(connection: &Connection) { + connection + .execute_batch( + "CREATE TABLE clone_body_payloads( + payload_digest TEXT PRIMARY KEY, + payload BLOB NOT NULL + ); + CREATE TABLE clone_occurrences( + symbol_occurrence_id TEXT PRIMARY KEY, + payload_digest TEXT NOT NULL, + occurrence BLOB NOT NULL + ); + CREATE TABLE clone_exact_postings( + class INTEGER NOT NULL, + digest TEXT NOT NULL, + symbol_occurrence_id TEXT NOT NULL + );", + ) + .expect("census schema"); + } + + fn payload(seed: usize) -> CloneBodyPayloadV1 { + let body = ExtractedCloneBodyV1 { + logical_path: format!("src/body_{seed}.rs"), + language: "rust".to_owned(), + symbol_kind: NodeKind::Function, + symbol_occurrence_id: format!("symbol.body.{seed}"), + body_span: SourceSpan { + start_byte: 0, + end_byte: 64, + }, + normalization_revision: 1, + non_trivia_token_count: 8, + eligibility: CloneBodyEligibilityV1::Eligible, + tokenization_status: CloneBodyTokenizationStatusV1::Complete, + tokenization_issues: Vec::new(), + conservative_tokens: Arc::from( + (0..8) + .map(|index| ConservativeCloneTokenV1::Syntax { + syntax_kind: "identifier".to_owned(), + text: format!("token_{seed}_{index}"), + }) + .collect::>(), + ), + rename_normalization_revision: None, + rename_status: CloneBodyRenameStatusV1::UnsupportedLanguage, + rename_issues: Vec::new(), + rename_tokens: None, + }; + CloneBodyPayloadV1::from_extracted(&body).expect("canonical clone payload") + } + + fn store_payload(connection: &Connection, payload: &CloneBodyPayloadV1) { + connection + .execute( + "INSERT INTO clone_body_payloads(payload_digest, payload) VALUES (?1, ?2)", + params![ + payload.payload_digest.as_str(), + serde_json::to_vec(payload).expect("payload json") + ], + ) + .expect("store payload"); + } + + fn store_occurrence(connection: &Connection, id: &str, payload: &CloneBodyPayloadV1) { + let occurrence = CloneBodyOccurrenceV1 { + project_id: ProjectId::new("project.clone-census").expect("project"), + repository_id: RepositoryId::new("repository.clone-census").expect("repository"), + worktree_id: None, + source_generation: CodeGenerationId::new("generation.clone-census") + .expect("generation"), + snapshot_digest: payload.body_digest.clone(), + symbol_occurrence_id: SymbolOccurrenceId::new(id).expect("symbol"), + path: "src/lib.rs".to_owned(), + body_span: SourceSpan { + start_byte: 0, + end_byte: 64, + }, + payload_digest: payload.payload_digest.clone(), + eligibility: CloneBodyEligibilityV1::Eligible, + }; + connection + .execute( + "INSERT INTO clone_occurrences(symbol_occurrence_id, payload_digest, occurrence) + VALUES (?1, ?2, ?3)", + params![ + id, + payload.payload_digest.as_str(), + serde_json::to_vec(&occurrence).expect("occurrence json") + ], + ) + .expect("store occurrence"); + } + + /// The census joins occurrences to payloads, so an occurrence whose payload + /// row is absent is invisible to the join. Counting it out of the totals is + /// an undercount reported as a healthy census, which is worse than a + /// refusal: the artifact is missing a row the occurrence says it holds. + #[test] + fn census_refuses_an_occurrence_whose_payload_row_is_absent() { + let connection = Connection::open_in_memory().expect("census database"); + census_schema(&connection); + let present = payload(0); + let absent = payload(1); + store_payload(&connection, &present); + store_occurrence(&connection, "symbol.present", &present); + store_occurrence(&connection, "symbol.absent", &absent); + + let error = read_clone_index_census(&connection, false, 8) + .expect_err("an occurrence without its payload row must refuse the census"); + assert!( + matches!(error, CodeLexicalArtifactErrorV1::Corrupt(_)), + "expected a corruption refusal, got {error:?}" + ); + } + + /// A census over a corpus whose occurrences share few bodies, which is the + /// shape every real repository has. + /// + /// Ignored because it reports a duration rather than asserting one; run it + /// with `--ignored --nocapture` to re-derive the figure in the module doc. + #[test] + #[ignore = "timing measurement, not a pass/fail contract"] + fn measure_read_clone_index_census() { + const PAYLOADS: usize = 128; + const OCCURRENCES_PER_PAYLOAD: usize = 768; + + let connection = Connection::open_in_memory().expect("census database"); + census_schema(&connection); + for seed in 0..PAYLOADS { + let payload = payload(seed); + store_payload(&connection, &payload); + for index in 0..OCCURRENCES_PER_PAYLOAD { + store_occurrence(&connection, &format!("symbol.{seed}.{index}"), &payload); + } + } + + let started = std::time::Instant::now(); + let census = read_clone_index_census(&connection, false, 8).expect("census"); + let elapsed = started.elapsed(); + assert_eq!( + census.source_bodies, + (PAYLOADS * OCCURRENCES_PER_PAYLOAD) as u64 + ); + println!( + "census over {} occurrences / {PAYLOADS} payloads: {:.3}s", + census.source_bodies, + elapsed.as_secs_f64() + ); + } +}