From 941d908ba92c043c561ff4d69a6f1370ca7c3d41 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 18 Sep 2026 14:07:24 +0000 Subject: [PATCH] fix(clones): bound body bytes before tokenizing large literals The 4,096-token exclusion still admitted a body containing one multi-megabyte literal because the literal is a single token. That record again exceeded the 4 MiB text page and parked graph seating. Enforce the original 64 KiB source-byte guard before tokenization alongside the token guard; carry both limits through contracts and UI, and retain no token streams for either exclusion. Move every extractor revision in the same cut so old poisoned artifacts are re-extracted. --- .../src/clone_body.rs | 40 ++++++++++++++++--- crates/tracedecay-code-extraction/src/lib.rs | 5 ++- .../tests/main/clone_body_tokens.rs | 36 +++++++++++++++-- .../code_index_scheduler/tests/reconcile.rs | 4 +- crates/tracedecay-code-index/src/extract.rs | 9 +++-- crates/tracedecay-code-index/src/languages.rs | 16 ++++---- .../src/production/worker_tests.rs | 4 +- .../src/retrieval/primitive_surface.rs | 9 ++++- .../src/handlers/graph/search.rs | 6 ++- .../tools/handlers/dashboard/code_reads.rs | 6 ++- .../schemas/dashboard-contracts.schema.json | 8 +++- dashboard/src/contracts/generated.ts | 1 + .../src/workspaces/code/sharedCode.test.ts | 9 +++++ dashboard/src/workspaces/code/sharedCode.ts | 2 +- sdks/typescript/src/operations.ts | 4 +- 15 files changed, 124 insertions(+), 35 deletions(-) diff --git a/crates/tracedecay-code-extraction/src/clone_body.rs b/crates/tracedecay-code-extraction/src/clone_body.rs index ef7510a6c5..62203922eb 100644 --- a/crates/tracedecay-code-extraction/src/clone_body.rs +++ b/crates/tracedecay-code-extraction/src/clone_body.rs @@ -12,6 +12,9 @@ mod rename; pub const CONSERVATIVE_CLONE_NORMALIZATION_REVISION_V1: u16 = 1; pub const RENAME_CLONE_NORMALIZATION_REVISION_V1: u16 = 1; pub const MIN_AUTOMATIC_CLONE_BODY_TOKENS_V1: u32 = 30; +/// Source-byte guard checked before tokenization. Token count cannot bound one +/// giant literal token, while the text artifact still serializes its bytes. +pub const MAX_AUTOMATIC_CLONE_BODY_BYTES_V1: u64 = 64 * 1024; /// Bodies with more non-trivia tokens than this are not clone candidates and /// keep no token stream. A clone body is persisted as one serialized record /// inside a 4 MiB text-artifact page; a 14k-token function (a generated @@ -35,8 +38,13 @@ pub enum ConservativeCloneTokenV1 { pub enum CloneBodyEligibilityV1 { Eligible, ExcludedIncompleteTokenization, - ExcludedTooSmall { minimum_tokens: u32 }, - ExcludedTooLarge { maximum_tokens: u32 }, + ExcludedTooSmall { + minimum_tokens: u32, + }, + ExcludedTooLarge { + maximum_tokens: u32, + maximum_bytes: u64, + }, } #[derive(Clone, Copy, Debug, Serialize, Deserialize, Eq, PartialEq, Ord, PartialOrd, Hash)] @@ -50,6 +58,7 @@ pub enum CloneBodyTokenizationStatusV1 { #[serde(rename_all = "snake_case")] pub enum CloneBodyTokenizationIssueV1 { BodyBoundaryUnavailable, + BodyExceedsSizeBound, InvalidSourceRange, ParseError, } @@ -176,7 +185,21 @@ fn extract_clone_body( language: &str, logical_path: &str, ) -> ExtractedCloneBodyV1 { - let conservative = conservative_fields(syntax, source, language); + let body_bytes = syntax + .body + .end_byte() + .saturating_sub(syntax.body.start_byte()) as u64; + let conservative = if body_bytes > MAX_AUTOMATIC_CLONE_BODY_BYTES_V1 { + ConservativeFields { + tokens: Arc::from([]), + issues: vec![CloneBodyTokenizationIssueV1::BodyExceedsSizeBound], + token_count: 0, + status: CloneBodyTokenizationStatusV1::Partial, + eligibility: oversized_clone_body(), + } + } else { + conservative_fields(syntax, source, language) + }; // An oversized body keeps its count and its typed exclusion but no // stream: the streams are what would not fit a page, and rename // normalization has nothing to normalize for. @@ -261,9 +284,7 @@ fn conservative_fields( minimum_tokens: MIN_AUTOMATIC_CLONE_BODY_TOKENS_V1, } } else if emitter.token_count > MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1 { - CloneBodyEligibilityV1::ExcludedTooLarge { - maximum_tokens: MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1, - } + oversized_clone_body() } else { CloneBodyEligibilityV1::Eligible }; @@ -281,6 +302,13 @@ fn conservative_fields( } } +const fn oversized_clone_body() -> CloneBodyEligibilityV1 { + CloneBodyEligibilityV1::ExcludedTooLarge { + maximum_tokens: MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1, + maximum_bytes: MAX_AUTOMATIC_CLONE_BODY_BYTES_V1, + } +} + struct RenameFields { revision: Option, status: CloneBodyRenameStatusV1, diff --git a/crates/tracedecay-code-extraction/src/lib.rs b/crates/tracedecay-code-extraction/src/lib.rs index 1db5516ffa..2662a6511d 100644 --- a/crates/tracedecay-code-extraction/src/lib.rs +++ b/crates/tracedecay-code-extraction/src/lib.rs @@ -128,8 +128,9 @@ pub use c_extractor::CExtractor; pub use clone_body::{ CONSERVATIVE_CLONE_NORMALIZATION_REVISION_V1, CloneBodyEligibilityV1, CloneBodyRenameIssueV1, CloneBodyRenameStatusV1, CloneBodyTokenizationIssueV1, CloneBodyTokenizationStatusV1, - ConservativeCloneTokenV1, ExtractedCloneBodyV1, MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1, - MIN_AUTOMATIC_CLONE_BODY_TOKENS_V1, RENAME_CLONE_NORMALIZATION_REVISION_V1, + ConservativeCloneTokenV1, ExtractedCloneBodyV1, MAX_AUTOMATIC_CLONE_BODY_BYTES_V1, + MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1, MIN_AUTOMATIC_CLONE_BODY_TOKENS_V1, + RENAME_CLONE_NORMALIZATION_REVISION_V1, }; pub use cpp_extractor::CppExtractor; pub use csharp_extractor::CSharpExtractor; diff --git a/crates/tracedecay-code-extraction/tests/main/clone_body_tokens.rs b/crates/tracedecay-code-extraction/tests/main/clone_body_tokens.rs index fb2ff5eca0..5d46ef6542 100644 --- a/crates/tracedecay-code-extraction/tests/main/clone_body_tokens.rs +++ b/crates/tracedecay-code-extraction/tests/main/clone_body_tokens.rs @@ -4,8 +4,8 @@ use tracedecay_code_extraction::ClojureExtractor; use tracedecay_code_extraction::PerlExtractor; use tracedecay_code_extraction::{ CloneBodyEligibilityV1, CloneBodyTokenizationIssueV1, CloneBodyTokenizationStatusV1, - ConservativeCloneTokenV1, LanguageExtractor, MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1, - PythonExtractor, RustExtractor, TypeScriptExtractor, + ConservativeCloneTokenV1, LanguageExtractor, MAX_AUTOMATIC_CLONE_BODY_BYTES_V1, + MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1, PythonExtractor, RustExtractor, TypeScriptExtractor, }; use tracedecay_domain::NodeKind; @@ -242,7 +242,8 @@ fn bodies_above_the_token_maximum_are_excluded_without_streams() { assert_eq!( body.eligibility, CloneBodyEligibilityV1::ExcludedTooLarge { - maximum_tokens: MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1 + maximum_tokens: MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1, + maximum_bytes: MAX_AUTOMATIC_CLONE_BODY_BYTES_V1, } ); assert!(body.conservative_tokens.is_empty()); @@ -260,6 +261,35 @@ fn bodies_above_the_token_maximum_are_excluded_without_streams() { assert!(!body.conservative_tokens.is_empty()); } +#[test] +fn body_bytes_are_bounded_before_a_large_literal_is_tokenized() { + let literal = "x".repeat(usize::try_from(MAX_AUTOMATIC_CLONE_BODY_BYTES_V1).unwrap()); + let artifact = RustExtractor.extract_artifact( + "src/lib.rs", + &format!("fn body() {{ let value = \"{literal}\"; }}"), + ); + let body = &artifact.clone_bodies[0]; + + assert_eq!( + body.eligibility, + CloneBodyEligibilityV1::ExcludedTooLarge { + maximum_tokens: MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1, + maximum_bytes: MAX_AUTOMATIC_CLONE_BODY_BYTES_V1, + } + ); + assert_eq!(body.non_trivia_token_count, 0); + assert!(body.conservative_tokens.is_empty()); + assert_eq!( + body.tokenization_issues, + vec![CloneBodyTokenizationIssueV1::BodyExceedsSizeBound] + ); + assert_eq!( + body.tokenization_status, + CloneBodyTokenizationStatusV1::Partial + ); + assert!(body.rename_tokens.is_none()); +} + #[test] fn clone_bodies_bind_to_method_and_stable_arrow_occurrences() { for (artifact, expected_kind, expected_language) in [ diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs index fe2022e54d..3fcf5b0e15 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs @@ -795,7 +795,7 @@ async fn restart_remount_serves_the_retained_generation_without_republishing() { } #[test] -fn retained_v3_rust_extractor_generation_is_refused_and_rebuilt_by_v5() { +fn retained_stale_rust_extractor_generation_is_refused_and_rebuilt() { let fixture = GitFixture::new(ALPHA_LIB_V1); let store = TempDir::new().expect("store root"); let mut seed = scheduler( @@ -836,7 +836,7 @@ fn retained_v3_rust_extractor_generation_is_refused_and_rebuilt_by_v5() { .iter() .find(|(language, _)| language.as_str() == "rust") .map(|(_, revision)| revision.as_str()), - Some("extractor.rust.v9") + Some("extractor.rust.v10") ); } diff --git a/crates/tracedecay-code-index/src/extract.rs b/crates/tracedecay-code-index/src/extract.rs index 3b92e43369..85eb041ceb 100644 --- a/crates/tracedecay-code-index/src/extract.rs +++ b/crates/tracedecay-code-index/src/extract.rs @@ -775,12 +775,13 @@ mod tests { // extractor.rust.v8 records restricted `pub` re-export scope as a // typed value and no longer fabricates receiver types for method - // initializers; v9 adds the clone-body token bound. The revision is - // part of the batch identity, so the pinned digest moves with it. - assert_eq!(descriptor.extractor_revision.as_str(), "extractor.rust.v9"); + // initializers; v9 adds the clone-body token bound and v10 the byte + // bound. The revision is part of the batch identity, so the pinned + // digest moves with it. + assert_eq!(descriptor.extractor_revision.as_str(), "extractor.rust.v10"); assert_eq!( extraction.batch().rows_digest.as_str(), - "sha256:9ec70789226ea2ea9d427d2da4b1c35b54a91da7cf839169b6104ddc2eb1afba" + "sha256:2e1ebb8fbd7b438059eda5da2db9c8db9707f219484a28e82066caf527b038c6" ); } diff --git a/crates/tracedecay-code-index/src/languages.rs b/crates/tracedecay-code-index/src/languages.rs index 641ade5762..9f3a01714c 100644 --- a/crates/tracedecay-code-index/src/languages.rs +++ b/crates/tracedecay-code-index/src/languages.rs @@ -208,16 +208,16 @@ impl StaticLanguageRegistry { // names), records restricted `pub` re-export scopes separately // from unrestricted exports, and resolves inherent impl methods across files of // the owning type. Every language moved one revision when clone-body - // eligibility gained its upper token bound: a sealed file artifact - // from before it can hold a body record no text-artifact page can - // admit, and only re-extraction removes it. Pinning these behaviors - // forces older file artifacts to be re-extracted. + // eligibility gained its token bound and again when it gained the + // pre-tokenization byte bound: one multi-megabyte literal is only + // a few tokens but still cannot fit a text-artifact page. Only + // re-extraction removes the poisoned record. let extractor_revision = if language == "rust" { - 9 + 10 } else if matches!(language.as_str(), "typescript" | "protobuf" | "sql") { - 5 + 6 } else { - 4 + 5 }; let descriptor = LanguageDescriptorV1 { language: LanguageId::new(language.clone()) @@ -418,7 +418,7 @@ mod tests { assert!(rust.stable_member_spans); assert!(rust.capabilities.extraction); assert_eq!(rust.root_markers, vec!["Cargo.toml".to_owned()]); - assert_eq!(rust.extractor_revision.as_str(), "extractor.rust.v9"); + assert_eq!(rust.extractor_revision.as_str(), "extractor.rust.v10"); assert_eq!( registry diff --git a/crates/tracedecay-code-index/src/production/worker_tests.rs b/crates/tracedecay-code-index/src/production/worker_tests.rs index 1dfd284f18..cda982bd16 100644 --- a/crates/tracedecay-code-index/src/production/worker_tests.rs +++ b/crates/tracedecay-code-index/src/production/worker_tests.rs @@ -357,7 +357,7 @@ fn extractor_revision_change_reextracts_before_validating_retained_import_rows() assert_eq!( rebuilt.files[0].extraction.extractor_revision.as_str(), - "extractor.rust.v9" + "extractor.rust.v10" ); assert_ne!( rebuilt.files[0].extraction.parser_import_rows_digest, @@ -425,7 +425,7 @@ fn physical_artifact_reuse_rejects_a_stale_extractor_revision() { assert_eq!( rebuilt.files[0].extraction.extractor_revision.as_str(), - "extractor.rust.v9" + "extractor.rust.v10" ); assert!( rebuilt.files[0] diff --git a/crates/tracedecay-contracts/src/retrieval/primitive_surface.rs b/crates/tracedecay-contracts/src/retrieval/primitive_surface.rs index a517c4c8c8..ad147d1c37 100644 --- a/crates/tracedecay-contracts/src/retrieval/primitive_surface.rs +++ b/crates/tracedecay-contracts/src/retrieval/primitive_surface.rs @@ -475,8 +475,13 @@ pub struct SimilarFamilyV1 { pub enum SimilarCoverageV1 { Complete, Partial, - ExcludedTooSmall { minimum_tokens: u32 }, - ExcludedTooLarge { maximum_tokens: u32 }, + ExcludedTooSmall { + minimum_tokens: u32, + }, + ExcludedTooLarge { + maximum_tokens: u32, + maximum_bytes: u64, + }, ExcludedIncompleteTokenization, } diff --git a/crates/tracedecay-mcp/src/handlers/graph/search.rs b/crates/tracedecay-mcp/src/handlers/graph/search.rs index d72a5db817..e6738a5e2e 100644 --- a/crates/tracedecay-mcp/src/handlers/graph/search.rs +++ b/crates/tracedecay-mcp/src/handlers/graph/search.rs @@ -1173,7 +1173,11 @@ pub async fn handle_similar(ctx: &McpToolContext<'_>, args: Value) -> Result SimilarCoverageV1::ExcludedTooSmall { minimum_tokens }, tracedecay_code_index::clones::CloneBodyEligibilityV1::ExcludedTooLarge { maximum_tokens, - } => SimilarCoverageV1::ExcludedTooLarge { maximum_tokens }, + maximum_bytes, + } => SimilarCoverageV1::ExcludedTooLarge { + maximum_tokens, + maximum_bytes, + }, tracedecay_code_index::clones::CloneBodyEligibilityV1::ExcludedIncompleteTokenization => { SimilarCoverageV1::ExcludedIncompleteTokenization } diff --git a/crates/tracedecay/src/mcp/tools/handlers/dashboard/code_reads.rs b/crates/tracedecay/src/mcp/tools/handlers/dashboard/code_reads.rs index aa7b205152..47245929a5 100644 --- a/crates/tracedecay/src/mcp/tools/handlers/dashboard/code_reads.rs +++ b/crates/tracedecay/src/mcp/tools/handlers/dashboard/code_reads.rs @@ -279,7 +279,11 @@ fn shared_family_result( } => SimilarCoverageV1::ExcludedTooSmall { minimum_tokens }, tracedecay_code_index::clones::CloneBodyEligibilityV1::ExcludedTooLarge { maximum_tokens, - } => SimilarCoverageV1::ExcludedTooLarge { maximum_tokens }, + maximum_bytes, + } => SimilarCoverageV1::ExcludedTooLarge { + maximum_tokens, + maximum_bytes, + }, tracedecay_code_index::clones::CloneBodyEligibilityV1::ExcludedIncompleteTokenization => { SimilarCoverageV1::ExcludedIncompleteTokenization } diff --git a/dashboard/codegen/schemas/dashboard-contracts.schema.json b/dashboard/codegen/schemas/dashboard-contracts.schema.json index 9149aeaf94..b1834bb6bc 100644 --- a/dashboard/codegen/schemas/dashboard-contracts.schema.json +++ b/dashboard/codegen/schemas/dashboard-contracts.schema.json @@ -21330,6 +21330,11 @@ { "additionalProperties": false, "properties": { + "maximum_bytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, "maximum_tokens": { "format": "uint32", "minimum": 0, @@ -21342,7 +21347,8 @@ }, "required": [ "status", - "maximum_tokens" + "maximum_tokens", + "maximum_bytes" ], "type": "object" }, diff --git a/dashboard/src/contracts/generated.ts b/dashboard/src/contracts/generated.ts index 2c1b32e6ec..49450c5d6b 100644 --- a/dashboard/src/contracts/generated.ts +++ b/dashboard/src/contracts/generated.ts @@ -5156,6 +5156,7 @@ export const SimilarCoverageV1Schema = z.discriminatedUnion("status", [z.object( }).strict(), z.object({ status: z.literal("excluded_incomplete_tokenization"), }).strict(), z.object({ + maximum_bytes: z.number().int().safe().min(0), maximum_tokens: z.number().int().min(0), status: z.literal("excluded_too_large"), }).strict(), z.object({ diff --git a/dashboard/src/workspaces/code/sharedCode.test.ts b/dashboard/src/workspaces/code/sharedCode.test.ts index c6f1bcf6bc..6d67c57aad 100644 --- a/dashboard/src/workspaces/code/sharedCode.test.ts +++ b/dashboard/src/workspaces/code/sharedCode.test.ts @@ -49,6 +49,15 @@ describe('shared-code coverage wording', () => { expect(tooSmall.sentence).toMatch(/30-token minimum/); expect(tooSmall.sentence).toMatch(/not a finding of zero copies/); + const tooLarge = readSharedCodeCoverage({ + status: 'excluded_too_large', + maximum_tokens: 4096, + maximum_bytes: 65536, + }); + expect(tooLarge.kind).toBe('excluded'); + if (tooLarge.kind !== 'excluded') throw new Error('unreachable'); + expect(tooLarge.sentence).toMatch(/4,096-token or 65,536-byte maximum/); + const incomplete = readSharedCodeCoverage({ status: 'excluded_incomplete_tokenization' }); expect(incomplete.kind).toBe('excluded'); if (incomplete.kind !== 'excluded') throw new Error('unreachable'); diff --git a/dashboard/src/workspaces/code/sharedCode.ts b/dashboard/src/workspaces/code/sharedCode.ts index 908d89eefc..2c8a25986b 100644 --- a/dashboard/src/workspaces/code/sharedCode.ts +++ b/dashboard/src/workspaces/code/sharedCode.ts @@ -100,7 +100,7 @@ export function readSharedCodeCoverage(coverage: SimilarCoverageV1): SharedCodeC return { kind: 'excluded', title: 'Excluded from automatic discovery', - sentence: `This body is over the ${coverage.maximum_tokens.toLocaleString()}-token maximum, so no family was searched for it. That is an exclusion, not a finding of zero copies.`, + sentence: `This body is over the automatic ${coverage.maximum_tokens.toLocaleString()}-token or ${coverage.maximum_bytes.toLocaleString()}-byte maximum, so no family was searched for it. That is an exclusion, not a finding of zero copies.`, }; case 'excluded_incomplete_tokenization': return { diff --git a/sdks/typescript/src/operations.ts b/sdks/typescript/src/operations.ts index 6bcd19d6cb..79b07f24a2 100644 --- a/sdks/typescript/src/operations.ts +++ b/sdks/typescript/src/operations.ts @@ -637,7 +637,7 @@ export type SettingKey = string; export type SettingSensitivityV1 = "public" | "sensitive"; export type SettingSummary = { readonly key: SettingKey; readonly restart_requirement: RestartRequirementV1; readonly sensitivity: SettingSensitivityV1; readonly [key: string]: unknown }; export type SharedProfileStoreLocatorV1 = { readonly brain_id: BrainId; readonly profile_id: UserProfileId; readonly store_id: string }; -export type SimilarCoverageV1 = { readonly status: "complete" } | { readonly status: "partial" } | { readonly minimum_tokens: number; readonly status: "excluded_too_small" } | { readonly maximum_tokens: number; readonly status: "excluded_too_large" } | { readonly status: "excluded_incomplete_tokenization" }; +export type SimilarCoverageV1 = { readonly status: "complete" } | { readonly status: "partial" } | { readonly minimum_tokens: number; readonly status: "excluded_too_small" } | { readonly maximum_bytes: number; readonly maximum_tokens: number; readonly status: "excluded_too_large" } | { readonly status: "excluded_incomplete_tokenization" }; export type SimilarFamilyV1 = { readonly complete: boolean; readonly family_digest: ManifestDigest; readonly match_class: SimilarMatchClassV1; readonly member_count: number; readonly members: readonly SimilarOccurrenceV1[]; readonly next_cursor?: string | null; readonly normalization_revision: number; readonly representative_payload_digest: ManifestDigest }; export type SimilarMatchClassV1 = "conservative_exact" | "rename_normalized_exact"; export type SimilarOccurrenceV1 = { readonly body_span: SourceSpan; readonly path: string; readonly project_id: ProjectId; readonly repository_id: RepositoryId; readonly snapshot_digest: ManifestDigest; readonly source_generation: CodeGenerationId; readonly symbol_occurrence_id: SymbolOccurrenceId; readonly worktree_id?: WorktreeId | null }; @@ -1909,7 +1909,7 @@ const DEFINITIONS = { SettingSensitivityV1: {"enum":["public","sensitive"],"type":"string"}, SettingSummary: {"properties":{"key":{"$ref":"#/$defs/SettingKey"},"restart_requirement":{"$ref":"#/$defs/RestartRequirementV1"},"sensitivity":{"$ref":"#/$defs/SettingSensitivityV1"}},"required":["key","sensitivity","restart_requirement"],"type":"object"}, SharedProfileStoreLocatorV1: {"additionalProperties":false,"description":"Logical and verified physical identity of the shared Profile shard.\n\nBrain and profile IDs select the shard; the verified store locator binds its\nphysical store. Lease incarnations and authority epochs remain runtime fences.","properties":{"brain_id":{"$ref":"#/$defs/BrainId"},"profile_id":{"$ref":"#/$defs/UserProfileId"},"store_id":{"type":"string"}},"required":["brain_id","profile_id","store_id"],"type":"object"}, - SimilarCoverageV1: {"oneOf":[{"additionalProperties":false,"properties":{"status":{"const":"complete","type":"string"}},"required":["status"],"type":"object"},{"additionalProperties":false,"properties":{"status":{"const":"partial","type":"string"}},"required":["status"],"type":"object"},{"additionalProperties":false,"properties":{"minimum_tokens":{"format":"uint32","minimum":0,"type":"integer"},"status":{"const":"excluded_too_small","type":"string"}},"required":["status","minimum_tokens"],"type":"object"},{"additionalProperties":false,"properties":{"maximum_tokens":{"format":"uint32","minimum":0,"type":"integer"},"status":{"const":"excluded_too_large","type":"string"}},"required":["status","maximum_tokens"],"type":"object"},{"additionalProperties":false,"properties":{"status":{"const":"excluded_incomplete_tokenization","type":"string"}},"required":["status"],"type":"object"}]}, + SimilarCoverageV1: {"oneOf":[{"additionalProperties":false,"properties":{"status":{"const":"complete","type":"string"}},"required":["status"],"type":"object"},{"additionalProperties":false,"properties":{"status":{"const":"partial","type":"string"}},"required":["status"],"type":"object"},{"additionalProperties":false,"properties":{"minimum_tokens":{"format":"uint32","minimum":0,"type":"integer"},"status":{"const":"excluded_too_small","type":"string"}},"required":["status","minimum_tokens"],"type":"object"},{"additionalProperties":false,"properties":{"maximum_bytes":{"format":"uint64","minimum":0,"type":"integer"},"maximum_tokens":{"format":"uint32","minimum":0,"type":"integer"},"status":{"const":"excluded_too_large","type":"string"}},"required":["status","maximum_tokens","maximum_bytes"],"type":"object"},{"additionalProperties":false,"properties":{"status":{"const":"excluded_incomplete_tokenization","type":"string"}},"required":["status"],"type":"object"}]}, SimilarFamilyV1: {"additionalProperties":false,"properties":{"complete":{"type":"boolean"},"family_digest":{"$ref":"#/$defs/ManifestDigest"},"match_class":{"$ref":"#/$defs/SimilarMatchClassV1"},"member_count":{"format":"uint","minimum":0,"type":"integer"},"members":{"items":{"$ref":"#/$defs/SimilarOccurrenceV1"},"type":"array"},"next_cursor":{"type":["string","null"]},"normalization_revision":{"format":"uint16","maximum":65535,"minimum":0,"type":"integer"},"representative_payload_digest":{"$ref":"#/$defs/ManifestDigest"}},"required":["match_class","normalization_revision","family_digest","representative_payload_digest","member_count","members","complete"],"type":"object"}, SimilarMatchClassV1: {"enum":["conservative_exact","rename_normalized_exact"],"type":"string"}, SimilarOccurrenceV1: {"additionalProperties":false,"properties":{"body_span":{"$ref":"#/$defs/SourceSpan"},"path":{"type":"string"},"project_id":{"$ref":"#/$defs/ProjectId"},"repository_id":{"$ref":"#/$defs/RepositoryId"},"snapshot_digest":{"$ref":"#/$defs/ManifestDigest"},"source_generation":{"$ref":"#/$defs/CodeGenerationId"},"symbol_occurrence_id":{"$ref":"#/$defs/SymbolOccurrenceId"},"worktree_id":{"anyOf":[{"$ref":"#/$defs/WorktreeId"},{"type":"null"}]}},"required":["project_id","repository_id","source_generation","snapshot_digest","symbol_occurrence_id","path","body_span"],"type":"object"},