fix(clones): bound body bytes before tokenizing large literals

crates/tracedecay-code-extraction/src/clone_body.rs

@@ -12,6 +12,9 @@ mod rename;
 pub const CONSERVATIVE_CLONE_NORMALIZATION_REVISION_V1: u16 = 1;
 pub const RENAME_CLONE_NORMALIZATION_REVISION_V1: u16 = 1;
 pub const MIN_AUTOMATIC_CLONE_BODY_TOKENS_V1: u32 = 30;
+/// Source-byte guard checked before tokenization. Token count cannot bound one
+/// giant literal token, while the text artifact still serializes its bytes.
+pub const MAX_AUTOMATIC_CLONE_BODY_BYTES_V1: u64 = 64 * 1024;
 /// Bodies with more non-trivia tokens than this are not clone candidates and
 /// keep no token stream. A clone body is persisted as one serialized record
 /// inside a 4 MiB text-artifact page; a 14k-token function (a generated
@@ -35,8 +38,13 @@ pub enum ConservativeCloneTokenV1 {
 pub enum CloneBodyEligibilityV1 {
     Eligible,
     ExcludedIncompleteTokenization,
-    ExcludedTooSmall { minimum_tokens: u32 },
-    ExcludedTooLarge { maximum_tokens: u32 },
+    ExcludedTooSmall {
+        minimum_tokens: u32,
+    },
+    ExcludedTooLarge {
+        maximum_tokens: u32,
+        maximum_bytes: u64,
+    },
 }
 
 #[derive(Clone, Copy, Debug, Serialize, Deserialize, Eq, PartialEq, Ord, PartialOrd, Hash)]
@@ -50,6 +58,7 @@ pub enum CloneBodyTokenizationStatusV1 {
 #[serde(rename_all = "snake_case")]
 pub enum CloneBodyTokenizationIssueV1 {
     BodyBoundaryUnavailable,
+    BodyExceedsSizeBound,
     InvalidSourceRange,
     ParseError,
 }
@@ -176,7 +185,21 @@ fn extract_clone_body(
     language: &str,
     logical_path: &str,
 ) -> ExtractedCloneBodyV1 {
-    let conservative = conservative_fields(syntax, source, language);
+    let body_bytes = syntax
+        .body
+        .end_byte()
+        .saturating_sub(syntax.body.start_byte()) as u64;
+    let conservative = if body_bytes > MAX_AUTOMATIC_CLONE_BODY_BYTES_V1 {
+        ConservativeFields {
+            tokens: Arc::from([]),
+            issues: vec![CloneBodyTokenizationIssueV1::BodyExceedsSizeBound],
+            token_count: 0,
+            status: CloneBodyTokenizationStatusV1::Partial,
+            eligibility: oversized_clone_body(),
+        }
+    } else {
+        conservative_fields(syntax, source, language)
+    };
     // An oversized body keeps its count and its typed exclusion but no
     // stream: the streams are what would not fit a page, and rename
     // normalization has nothing to normalize for.
@@ -261,9 +284,7 @@ fn conservative_fields(
             minimum_tokens: MIN_AUTOMATIC_CLONE_BODY_TOKENS_V1,
         }
     } else if emitter.token_count > MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1 {
-        CloneBodyEligibilityV1::ExcludedTooLarge {
-            maximum_tokens: MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1,
-        }
+        oversized_clone_body()
     } else {
         CloneBodyEligibilityV1::Eligible
     };
@@ -281,6 +302,13 @@ fn conservative_fields(
     }
 }
 
+const fn oversized_clone_body() -> CloneBodyEligibilityV1 {
+    CloneBodyEligibilityV1::ExcludedTooLarge {
+        maximum_tokens: MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1,
+        maximum_bytes: MAX_AUTOMATIC_CLONE_BODY_BYTES_V1,
+    }
+}
+
 struct RenameFields {
     revision: Option<u16>,
     status: CloneBodyRenameStatusV1,

crates/tracedecay-code-extraction/src/lib.rs

@@ -128,8 +128,9 @@ pub use c_extractor::CExtractor;
 pub use clone_body::{
     CONSERVATIVE_CLONE_NORMALIZATION_REVISION_V1, CloneBodyEligibilityV1, CloneBodyRenameIssueV1,
     CloneBodyRenameStatusV1, CloneBodyTokenizationIssueV1, CloneBodyTokenizationStatusV1,
-    ConservativeCloneTokenV1, ExtractedCloneBodyV1, MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1,
-    MIN_AUTOMATIC_CLONE_BODY_TOKENS_V1, RENAME_CLONE_NORMALIZATION_REVISION_V1,
+    ConservativeCloneTokenV1, ExtractedCloneBodyV1, MAX_AUTOMATIC_CLONE_BODY_BYTES_V1,
+    MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1, MIN_AUTOMATIC_CLONE_BODY_TOKENS_V1,
+    RENAME_CLONE_NORMALIZATION_REVISION_V1,
 };
 pub use cpp_extractor::CppExtractor;
 pub use csharp_extractor::CSharpExtractor;

crates/tracedecay-code-extraction/tests/main/clone_body_tokens.rs

@@ -4,8 +4,8 @@ use tracedecay_code_extraction::ClojureExtractor;
 use tracedecay_code_extraction::PerlExtractor;
 use tracedecay_code_extraction::{
     CloneBodyEligibilityV1, CloneBodyTokenizationIssueV1, CloneBodyTokenizationStatusV1,
-    ConservativeCloneTokenV1, LanguageExtractor, MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1,
-    PythonExtractor, RustExtractor, TypeScriptExtractor,
+    ConservativeCloneTokenV1, LanguageExtractor, MAX_AUTOMATIC_CLONE_BODY_BYTES_V1,
+    MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1, PythonExtractor, RustExtractor, TypeScriptExtractor,
 };
 use tracedecay_domain::NodeKind;
 
@@ -242,7 +242,8 @@ fn bodies_above_the_token_maximum_are_excluded_without_streams() {
     assert_eq!(
         body.eligibility,
         CloneBodyEligibilityV1::ExcludedTooLarge {
-            maximum_tokens: MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1
+            maximum_tokens: MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1,
+            maximum_bytes: MAX_AUTOMATIC_CLONE_BODY_BYTES_V1,
         }
     );
     assert!(body.conservative_tokens.is_empty());
@@ -260,6 +261,35 @@ fn bodies_above_the_token_maximum_are_excluded_without_streams() {
     assert!(!body.conservative_tokens.is_empty());
 }
 
+#[test]
+fn body_bytes_are_bounded_before_a_large_literal_is_tokenized() {
+    let literal = "x".repeat(usize::try_from(MAX_AUTOMATIC_CLONE_BODY_BYTES_V1).unwrap());
+    let artifact = RustExtractor.extract_artifact(
+        "src/lib.rs",
+        &format!("fn body() {{ let value = \"{literal}\"; }}"),
+    );
+    let body = &artifact.clone_bodies[0];
+
+    assert_eq!(
+        body.eligibility,
+        CloneBodyEligibilityV1::ExcludedTooLarge {
+            maximum_tokens: MAX_AUTOMATIC_CLONE_BODY_TOKENS_V1,
+            maximum_bytes: MAX_AUTOMATIC_CLONE_BODY_BYTES_V1,
+        }
+    );
+    assert_eq!(body.non_trivia_token_count, 0);
+    assert!(body.conservative_tokens.is_empty());
+    assert_eq!(
+        body.tokenization_issues,
+        vec![CloneBodyTokenizationIssueV1::BodyExceedsSizeBound]
+    );
+    assert_eq!(
+        body.tokenization_status,
+        CloneBodyTokenizationStatusV1::Partial
+    );
+    assert!(body.rename_tokens.is_none());
+}
+
 #[test]
 fn clone_bodies_bind_to_method_and_stable_arrow_occurrences() {
     for (artifact, expected_kind, expected_language) in [

crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/reconcile.rs

@@ -795,7 +795,7 @@ async fn restart_remount_serves_the_retained_generation_without_republishing() {
 }
 
 #[test]
-fn retained_v3_rust_extractor_generation_is_refused_and_rebuilt_by_v5() {
+fn retained_stale_rust_extractor_generation_is_refused_and_rebuilt() {
     let fixture = GitFixture::new(ALPHA_LIB_V1);
     let store = TempDir::new().expect("store root");
     let mut seed = scheduler(
@@ -836,7 +836,7 @@ fn retained_v3_rust_extractor_generation_is_refused_and_rebuilt_by_v5() {
             .iter()
             .find(|(language, _)| language.as_str() == "rust")
             .map(|(_, revision)| revision.as_str()),
-        Some("extractor.rust.v9")
+        Some("extractor.rust.v10")
     );
 }
 

crates/tracedecay-code-index/src/extract.rs

@@ -775,12 +775,13 @@ mod tests {
 
         // extractor.rust.v8 records restricted `pub` re-export scope as a
         // typed value and no longer fabricates receiver types for method
-        // initializers; v9 adds the clone-body token bound. The revision is
-        // part of the batch identity, so the pinned digest moves with it.
-        assert_eq!(descriptor.extractor_revision.as_str(), "extractor.rust.v9");
+        // initializers; v9 adds the clone-body token bound and v10 the byte
+        // bound. The revision is part of the batch identity, so the pinned
+        // digest moves with it.
+        assert_eq!(descriptor.extractor_revision.as_str(), "extractor.rust.v10");
         assert_eq!(
             extraction.batch().rows_digest.as_str(),
-            "sha256:9ec70789226ea2ea9d427d2da4b1c35b54a91da7cf839169b6104ddc2eb1afba"
+            "sha256:2e1ebb8fbd7b438059eda5da2db9c8db9707f219484a28e82066caf527b038c6"
         );
     }
 

crates/tracedecay-code-index/src/languages.rs

@@ -208,16 +208,16 @@ impl StaticLanguageRegistry {
             // names), records restricted `pub` re-export scopes separately
             // from unrestricted exports, and resolves inherent impl methods across files of
             // the owning type. Every language moved one revision when clone-body
-            // eligibility gained its upper token bound: a sealed file artifact
-            // from before it can hold a body record no text-artifact page can
-            // admit, and only re-extraction removes it. Pinning these behaviors
-            // forces older file artifacts to be re-extracted.
+            // eligibility gained its token bound and again when it gained the
+            // pre-tokenization byte bound: one multi-megabyte literal is only
+            // a few tokens but still cannot fit a text-artifact page. Only
+            // re-extraction removes the poisoned record.
             let extractor_revision = if language == "rust" {
-                9
+                10
             } else if matches!(language.as_str(), "typescript" | "protobuf" | "sql") {
-                5
+                6
             } else {
-                4
+                5
             };
             let descriptor = LanguageDescriptorV1 {
                 language: LanguageId::new(language.clone())
@@ -418,7 +418,7 @@ mod tests {
         assert!(rust.stable_member_spans);
         assert!(rust.capabilities.extraction);
         assert_eq!(rust.root_markers, vec!["Cargo.toml".to_owned()]);
-        assert_eq!(rust.extractor_revision.as_str(), "extractor.rust.v9");
+        assert_eq!(rust.extractor_revision.as_str(), "extractor.rust.v10");
 
         assert_eq!(
             registry

crates/tracedecay-code-index/src/production/worker_tests.rs

@@ -357,7 +357,7 @@ fn extractor_revision_change_reextracts_before_validating_retained_import_rows()
 
     assert_eq!(
         rebuilt.files[0].extraction.extractor_revision.as_str(),
-        "extractor.rust.v9"
+        "extractor.rust.v10"
     );
     assert_ne!(
         rebuilt.files[0].extraction.parser_import_rows_digest,
@@ -425,7 +425,7 @@ fn physical_artifact_reuse_rejects_a_stale_extractor_revision() {
 
     assert_eq!(
         rebuilt.files[0].extraction.extractor_revision.as_str(),
-        "extractor.rust.v9"
+        "extractor.rust.v10"
     );
     assert!(
         rebuilt.files[0]

crates/tracedecay-contracts/src/retrieval/primitive_surface.rs

@@ -475,8 +475,13 @@ pub struct SimilarFamilyV1 {
 pub enum SimilarCoverageV1 {
     Complete,
     Partial,
-    ExcludedTooSmall { minimum_tokens: u32 },
-    ExcludedTooLarge { maximum_tokens: u32 },
+    ExcludedTooSmall {
+        minimum_tokens: u32,
+    },
+    ExcludedTooLarge {
+        maximum_tokens: u32,
+        maximum_bytes: u64,
+    },
     ExcludedIncompleteTokenization,
 }
 

crates/tracedecay-mcp/src/handlers/graph/search.rs

@@ -1173,7 +1173,11 @@ pub async fn handle_similar(ctx: &McpToolContext<'_>, args: Value) -> Result<Too
         } => SimilarCoverageV1::ExcludedTooSmall { minimum_tokens },
         tracedecay_code_index::clones::CloneBodyEligibilityV1::ExcludedTooLarge {
             maximum_tokens,
-        } => SimilarCoverageV1::ExcludedTooLarge { maximum_tokens },
+            maximum_bytes,
+        } => SimilarCoverageV1::ExcludedTooLarge {
+            maximum_tokens,
+            maximum_bytes,
+        },
         tracedecay_code_index::clones::CloneBodyEligibilityV1::ExcludedIncompleteTokenization => {
             SimilarCoverageV1::ExcludedIncompleteTokenization
         }

crates/tracedecay/src/mcp/tools/handlers/dashboard/code_reads.rs

@@ -279,7 +279,11 @@ fn shared_family_result(
         } => SimilarCoverageV1::ExcludedTooSmall { minimum_tokens },
         tracedecay_code_index::clones::CloneBodyEligibilityV1::ExcludedTooLarge {
             maximum_tokens,
-        } => SimilarCoverageV1::ExcludedTooLarge { maximum_tokens },
+            maximum_bytes,
+        } => SimilarCoverageV1::ExcludedTooLarge {
+            maximum_tokens,
+            maximum_bytes,
+        },
         tracedecay_code_index::clones::CloneBodyEligibilityV1::ExcludedIncompleteTokenization => {
             SimilarCoverageV1::ExcludedIncompleteTokenization
         }

dashboard/src/workspaces/code/sharedCode.test.ts

@@ -49,6 +49,15 @@ describe('shared-code coverage wording', () => {
     expect(tooSmall.sentence).toMatch(/30-token minimum/);
     expect(tooSmall.sentence).toMatch(/not a finding of zero copies/);
 
+    const tooLarge = readSharedCodeCoverage({
+      status: 'excluded_too_large',
+      maximum_tokens: 4096,
+      maximum_bytes: 65536,
+    });
+    expect(tooLarge.kind).toBe('excluded');
+    if (tooLarge.kind !== 'excluded') throw new Error('unreachable');
+    expect(tooLarge.sentence).toMatch(/4,096-token or 65,536-byte maximum/);
+
     const incomplete = readSharedCodeCoverage({ status: 'excluded_incomplete_tokenization' });
     expect(incomplete.kind).toBe('excluded');
     if (incomplete.kind !== 'excluded') throw new Error('unreachable');

dashboard/src/workspaces/code/sharedCode.ts

@@ -100,7 +100,7 @@ export function readSharedCodeCoverage(coverage: SimilarCoverageV1): SharedCodeC
       return {
         kind: 'excluded',
         title: 'Excluded from automatic discovery',
-        sentence: `This body is over the ${coverage.maximum_tokens.toLocaleString()}-token maximum, so no family was searched for it. That is an exclusion, not a finding of zero copies.`,
+        sentence: `This body is over the automatic ${coverage.maximum_tokens.toLocaleString()}-token or ${coverage.maximum_bytes.toLocaleString()}-byte maximum, so no family was searched for it. That is an exclusion, not a finding of zero copies.`,
       };
     case 'excluded_incomplete_tokenization':
       return {