diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs index b10ffc58a8..1c57277969 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs @@ -269,7 +269,23 @@ fn refs_heads_signature(dir: &Path) -> Option { /// Resolve the git-dir (worktree-local) and common-dir (repository-shared) /// paths, falling back to `/.git` when gix cannot open the checkout so a /// non-repository path still yields a stable, if empty, fingerprint. +/// +/// These two paths are structural, but the fingerprint above is sampled on +/// every query admission, and re-deriving them through a fresh repository open +/// dominates its cost, against the retained topology this now asks first. A +/// checkout that carries `/.git` is one an open at exactly this root +/// resolves through, which is also where a discovery started at this root +/// stops, so the retained answer is the same answer. Anything else, a bare +/// repository's control directory or a path that is not a checkout root, still +/// opens directly, because discovery would walk past it to an ancestor whose +/// git metadata does not describe this project. fn git_metadata_dirs(project_root: &Path) -> (PathBuf, PathBuf) { + if project_root.join(".git").exists() + && let Ok(topology) = + tracedecay_runtime_core::git_repository::repository_topology(project_root) + { + return (topology.git_dir.clone(), topology.common_dir.clone()); + } if let Ok(repository) = gix::open(project_root) { let git_dir = repository.git_dir().to_path_buf(); let common_dir = { @@ -487,4 +503,26 @@ mod tests { "an in-place loose-ref rewrite must be detected by the refs signature" ); } + + /// Cost of one tier-1 staleness sample, the number this module's contract + /// calls fixed and cheap. + /// + /// Ignored because it reports a duration rather than asserting one; run it + /// with `--ignored --nocapture` to re-derive the figure in the module doc. + #[test] + #[ignore = "timing measurement, not a pass/fail contract"] + fn measure_git_metadata_fingerprint_capture() { + const ITERATIONS: u32 = 2000; + + let repo = init_repo(&[("src/lib.rs", "pub fn a() {}\n")]); + for _ in 0..100 { + std::hint::black_box(GitMetadataFingerprintV1::capture(repo.path())); + } + let started = std::time::Instant::now(); + for _ in 0..ITERATIONS { + std::hint::black_box(GitMetadataFingerprintV1::capture(repo.path())); + } + let per_capture = started.elapsed() / ITERATIONS; + println!("capture: {:.1}us", per_capture.as_secs_f64() * 1e6); + } } diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_census.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_census.rs index a6661692a7..967872c365 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_census.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/clone_census.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use rusqlite::Connection; use tracedecay_code_index::clones::{ CloneBodyEligibilityV1, CloneBodyOccurrenceV1, CloneBodyPayloadV1, CloneBodyRenameStatusV1, @@ -25,15 +27,70 @@ pub struct CodeLexicalCloneIndexCensusV1 { pub rename_unsupported_bodies: u64, } +/// The rename coverages the per-occurrence counters distinguish. `Complete` +/// bumps no counter, so it is never retained and the census cannot hold a +/// coverage it would then have to drop. +#[derive(Clone, Copy)] +enum IncompleteRenameCoverageV1 { + Partial, + UnsupportedLanguage, +} + +/// Validate every stored clone payload once and report which of them lack +/// complete rename normalization. +/// +/// `clone_body_payloads` is keyed by payload digest, so one row backs every +/// occurrence that shares that body. Re-deriving a payload's four canonical +/// digests once per occurrence therefore repeated the same verification for +/// every duplicate: a generated 768-file corpus stores 98,304 occurrences over +/// 128 distinct payloads, and the census spent ~1.2 s of single-threaded +/// verification where 128 validations were the whole obligation. +/// +/// Only the non-`Complete` coverages are retained, because those are the only +/// ones the per-occurrence rename counters distinguish. +fn validate_stored_clone_payloads( + connection: &Connection, +) -> Result, CodeLexicalArtifactErrorV1> { + let mut incomplete_rename = HashMap::new(); + let mut statement = connection + .prepare("SELECT payload_digest, payload FROM clone_body_payloads ORDER BY payload_digest") + .map_err(sqlite_error)?; + let mut rows = statement.query([]).map_err(sqlite_error)?; + while let Some(row) = rows.next().map_err(sqlite_error)? { + let digest: String = row.get(0).map_err(sqlite_error)?; + let payload_bytes: Vec = row.get(1).map_err(sqlite_error)?; + let payload: CloneBodyPayloadV1 = serde_json::from_slice(&payload_bytes) + .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; + if payload.payload_digest.as_str() != digest || payload.validate().is_err() { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "clone census found a payload outside its stored digest".to_owned(), + )); + } + match payload.rename_coverage { + CloneBodyRenameStatusV1::Complete => {} + CloneBodyRenameStatusV1::Partial => { + incomplete_rename.insert(digest, IncompleteRenameCoverageV1::Partial); + } + CloneBodyRenameStatusV1::UnsupportedLanguage => { + incomplete_rename.insert(digest, IncompleteRenameCoverageV1::UnsupportedLanguage); + } + } + } + Ok(incomplete_rename) +} + pub(super) fn read_clone_index_census( connection: &Connection, has_fingerprints: bool, hot_posting_threshold: u64, ) -> Result { let mut census = CodeLexicalCloneIndexCensusV1::default(); + let incomplete_rename = validate_stored_clone_payloads(connection)?; + // The inner join proves every counted occurrence has its verified payload + // row; the occurrence total below proves none was dropped by it. let mut statement = connection .prepare( - "SELECT occurrence.occurrence, payload.payload + "SELECT occurrence.payload_digest, occurrence.occurrence FROM clone_occurrences AS occurrence JOIN clone_body_payloads AS payload ON payload.payload_digest = occurrence.payload_digest @@ -42,13 +99,11 @@ pub(super) fn read_clone_index_census( .map_err(sqlite_error)?; let mut rows = statement.query([]).map_err(sqlite_error)?; while let Some(row) = rows.next().map_err(sqlite_error)? { - let occurrence_bytes: Vec = row.get(0).map_err(sqlite_error)?; - let payload_bytes: Vec = row.get(1).map_err(sqlite_error)?; + let digest: String = row.get(0).map_err(sqlite_error)?; + let occurrence_bytes: Vec = row.get(1).map_err(sqlite_error)?; let occurrence: CloneBodyOccurrenceV1 = serde_json::from_slice(&occurrence_bytes) .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; - let payload: CloneBodyPayloadV1 = serde_json::from_slice(&payload_bytes) - .map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string()))?; - if occurrence.payload_digest != payload.payload_digest || payload.validate().is_err() { + if occurrence.payload_digest.as_str() != digest { return Err(CodeLexicalArtifactErrorV1::Corrupt( "clone census found a payload outside its occurrence binding".to_owned(), )); @@ -57,13 +112,13 @@ pub(super) fn read_clone_index_census( match occurrence.eligibility { CloneBodyEligibilityV1::Eligible => { census.eligible_source_bodies = census.eligible_source_bodies.saturating_add(1); - match payload.rename_coverage { - CloneBodyRenameStatusV1::Complete => {} - CloneBodyRenameStatusV1::Partial => { + match incomplete_rename.get(&digest) { + None => {} + Some(IncompleteRenameCoverageV1::Partial) => { census.rename_partial_bodies = census.rename_partial_bodies.saturating_add(1); } - CloneBodyRenameStatusV1::UnsupportedLanguage => { + Some(IncompleteRenameCoverageV1::UnsupportedLanguage) => { census.rename_unsupported_bodies = census.rename_unsupported_bodies.saturating_add(1); } @@ -87,23 +142,43 @@ pub(super) fn read_clone_index_census( drop(rows); drop(statement); - let (unique_payloads, exact_postings, conservative, rename): (i64, i64, i64, i64) = connection + let (unique_payloads, exact_postings, conservative, rename, occurrences): ( + i64, + i64, + i64, + i64, + i64, + ) = connection .query_row( "SELECT (SELECT COUNT(*) FROM clone_body_payloads), (SELECT COUNT(*) FROM clone_exact_postings), (SELECT COUNT(*) FROM clone_exact_postings WHERE class = ?1), - (SELECT COUNT(*) FROM clone_exact_postings WHERE class = ?2)", + (SELECT COUNT(*) FROM clone_exact_postings WHERE class = ?2), + (SELECT COUNT(*) FROM clone_occurrences)", [ i64::from(CloneNormalizationClassV1::Conservative as u8), i64::from(CloneNormalizationClassV1::Rename as u8), ], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }, ) .map_err(sqlite_error)?; let count = |value: i64| -> Result { u64::try_from(value).map_err(|error| CodeLexicalArtifactErrorV1::Corrupt(error.to_string())) }; + if count(occurrences)? != census.source_bodies { + return Err(CodeLexicalArtifactErrorV1::Corrupt( + "clone census found an occurrence without its payload".to_owned(), + )); + } census.unique_payloads = count(unique_payloads)?; census.exact_postings = count(exact_postings)?; census.conservative_normalized_bodies = count(conservative)?; @@ -135,3 +210,169 @@ pub(super) fn read_clone_index_census( } Ok(census) } + +#[cfg(test)] +mod tests { + use super::*; + use rusqlite::params; + use std::sync::Arc; + use tracedecay_code_extraction::{ + CloneBodyTokenizationStatusV1, ConservativeCloneTokenV1, ExtractedCloneBodyV1, + }; + use tracedecay_domain::{ + CodeGenerationId, NodeKind, ProjectId, RepositoryId, SourceSpan, SymbolOccurrenceId, + }; + + /// The three tables the census reads. Triggers and the builder gate belong + /// to the write path, which no census read goes through. + fn census_schema(connection: &Connection) { + connection + .execute_batch( + "CREATE TABLE clone_body_payloads( + payload_digest TEXT PRIMARY KEY, + payload BLOB NOT NULL + ); + CREATE TABLE clone_occurrences( + symbol_occurrence_id TEXT PRIMARY KEY, + payload_digest TEXT NOT NULL, + occurrence BLOB NOT NULL + ); + CREATE TABLE clone_exact_postings( + class INTEGER NOT NULL, + digest TEXT NOT NULL, + symbol_occurrence_id TEXT NOT NULL + );", + ) + .expect("census schema"); + } + + fn payload(seed: usize) -> CloneBodyPayloadV1 { + let body = ExtractedCloneBodyV1 { + logical_path: format!("src/body_{seed}.rs"), + language: "rust".to_owned(), + symbol_kind: NodeKind::Function, + symbol_occurrence_id: format!("symbol.body.{seed}"), + body_span: SourceSpan { + start_byte: 0, + end_byte: 64, + }, + normalization_revision: 1, + non_trivia_token_count: 8, + eligibility: CloneBodyEligibilityV1::Eligible, + tokenization_status: CloneBodyTokenizationStatusV1::Complete, + tokenization_issues: Vec::new(), + conservative_tokens: Arc::from( + (0..8) + .map(|index| ConservativeCloneTokenV1::Syntax { + syntax_kind: "identifier".to_owned(), + text: format!("token_{seed}_{index}"), + }) + .collect::>(), + ), + rename_normalization_revision: None, + rename_status: CloneBodyRenameStatusV1::UnsupportedLanguage, + rename_issues: Vec::new(), + rename_tokens: None, + }; + CloneBodyPayloadV1::from_extracted(&body).expect("canonical clone payload") + } + + fn store_payload(connection: &Connection, payload: &CloneBodyPayloadV1) { + connection + .execute( + "INSERT INTO clone_body_payloads(payload_digest, payload) VALUES (?1, ?2)", + params![ + payload.payload_digest.as_str(), + serde_json::to_vec(payload).expect("payload json") + ], + ) + .expect("store payload"); + } + + fn store_occurrence(connection: &Connection, id: &str, payload: &CloneBodyPayloadV1) { + let occurrence = CloneBodyOccurrenceV1 { + project_id: ProjectId::new("project.clone-census").expect("project"), + repository_id: RepositoryId::new("repository.clone-census").expect("repository"), + worktree_id: None, + source_generation: CodeGenerationId::new("generation.clone-census") + .expect("generation"), + snapshot_digest: payload.body_digest.clone(), + symbol_occurrence_id: SymbolOccurrenceId::new(id).expect("symbol"), + path: "src/lib.rs".to_owned(), + body_span: SourceSpan { + start_byte: 0, + end_byte: 64, + }, + payload_digest: payload.payload_digest.clone(), + eligibility: CloneBodyEligibilityV1::Eligible, + }; + connection + .execute( + "INSERT INTO clone_occurrences(symbol_occurrence_id, payload_digest, occurrence) + VALUES (?1, ?2, ?3)", + params![ + id, + payload.payload_digest.as_str(), + serde_json::to_vec(&occurrence).expect("occurrence json") + ], + ) + .expect("store occurrence"); + } + + /// The census joins occurrences to payloads, so an occurrence whose payload + /// row is absent is invisible to the join. Counting it out of the totals is + /// an undercount reported as a healthy census, which is worse than a + /// refusal: the artifact is missing a row the occurrence says it holds. + #[test] + fn census_refuses_an_occurrence_whose_payload_row_is_absent() { + let connection = Connection::open_in_memory().expect("census database"); + census_schema(&connection); + let present = payload(0); + let absent = payload(1); + store_payload(&connection, &present); + store_occurrence(&connection, "symbol.present", &present); + store_occurrence(&connection, "symbol.absent", &absent); + + let error = read_clone_index_census(&connection, false, 8) + .expect_err("an occurrence without its payload row must refuse the census"); + assert!( + matches!(error, CodeLexicalArtifactErrorV1::Corrupt(_)), + "expected a corruption refusal, got {error:?}" + ); + } + + /// A census over a corpus whose occurrences share few bodies, which is the + /// shape every real repository has. + /// + /// Ignored because it reports a duration rather than asserting one; run it + /// with `--ignored --nocapture` to re-derive the figure in the module doc. + #[test] + #[ignore = "timing measurement, not a pass/fail contract"] + fn measure_read_clone_index_census() { + const PAYLOADS: usize = 128; + const OCCURRENCES_PER_PAYLOAD: usize = 768; + + let connection = Connection::open_in_memory().expect("census database"); + census_schema(&connection); + for seed in 0..PAYLOADS { + let payload = payload(seed); + store_payload(&connection, &payload); + for index in 0..OCCURRENCES_PER_PAYLOAD { + store_occurrence(&connection, &format!("symbol.{seed}.{index}"), &payload); + } + } + + let started = std::time::Instant::now(); + let census = read_clone_index_census(&connection, false, 8).expect("census"); + let elapsed = started.elapsed(); + assert_eq!( + census.source_bodies, + (PAYLOADS * OCCURRENCES_PER_PAYLOAD) as u64 + ); + println!( + "census over {} occurrences / {PAYLOADS} payloads: {:.3}s", + census.source_bodies, + elapsed.as_secs_f64() + ); + } +}