refactor(extraction): decode documents from required buffers

src/eslint-suppressions.json

@@ -1184,11 +1184,6 @@
 			"count": 1
 		}
 	},
-	"integrations/misc/__tests__/extract-text.spec.ts": {
-		"@typescript-eslint/no-explicit-any": {
-			"count": 4
-		}
-	},
 	"integrations/misc/__tests__/line-counter.spec.ts": {
 		"@typescript-eslint/no-explicit-any": {
 			"count": 2

src/integrations/misc/__tests__/extract-text-formats.integration.test.ts

@@ -0,0 +1,49 @@
+import fs from "fs/promises"
+import path from "path"
+import { createHash } from "node:crypto"
+import { extractTextFromBuffer } from "../extract-text"
+
+// Keep real parser fixtures separate from the mocked parser unit suite.
+describe("real document buffer decoding", () => {
+	afterEach(() => vi.restoreAllMocks())
+
+	it.each([
+		{
+			filename: "approved.pdf",
+			sha256: "11b19aed8bc2e40b9eda9d18b7bfe36e0bbc41c4383d7e30141927992790f411",
+		},
+		{
+			filename: "approved.docx",
+			sha256: "732d827ad38ff51c24983c773bcb964ba53ca29bb055ea73aa9fbed820ae3a9b",
+		},
+	])("matches the documented SHA-256 for $filename", async ({ filename, sha256 }) => {
+		const source = await fs.readFile(path.join(__dirname, "fixtures", filename))
+		expect(createHash("sha256").update(source).digest("hex")).toBe(sha256)
+	})
+
+	it("extracts numbered PDF text from fixture bytes without reopening the metadata pathname", async () => {
+		const source = await fs.readFile(path.join(__dirname, "fixtures", "approved.pdf"))
+		const access = vi.spyOn(fs, "access").mockRejectedValue(new Error("Unexpected pathname access"))
+		const readFile = vi.spyOn(fs, "readFile").mockRejectedValue(new Error("Unexpected pathname read"))
+
+		// pdf-parse prefixes each page with two newlines, which retain their line numbers.
+		expect(await extractTextFromBuffer(source, "metadata-only.PDF")).toBe(
+			"1 | \n2 | \n3 | Approved PDF first line\n4 | Approved PDF second line\n",
+		)
+		expect(access).not.toHaveBeenCalled()
+		expect(readFile).not.toHaveBeenCalled()
+	})
+
+	it("extracts numbered DOCX paragraphs from fixture bytes without reopening the metadata pathname", async () => {
+		const source = await fs.readFile(path.join(__dirname, "fixtures", "approved.docx"))
+		const access = vi.spyOn(fs, "access").mockRejectedValue(new Error("Unexpected pathname access"))
+		const readFile = vi.spyOn(fs, "readFile").mockRejectedValue(new Error("Unexpected pathname read"))
+
+		// Mammoth separates paragraphs with two newlines and decodes XML entities across runs.
+		expect(await extractTextFromBuffer(source, "metadata-only.DOCX")).toBe(
+			"1 | Approved DOCX first paragraph\n2 | \n3 | Approved DOCX second & final paragraph\n4 | \n",
+		)
+		expect(access).not.toHaveBeenCalled()
+		expect(readFile).not.toHaveBeenCalled()
+	})
+})

src/integrations/misc/__tests__/extract-text-from-xlsx.test.ts

@@ -1,9 +1,50 @@
 import ExcelJS from "exceljs"
-import { extractTextFromXLSX } from "../extract-text-from-xlsx"
+import { extractTextFromXLSX, formatWorkbook } from "../extract-text-from-xlsx"
+import { extractTextFromFile } from "../extract-text"
+
+describe("XLSX buffer decoding", () => {
+	it("preserves empty columns in a populated row from real XLSX bytes", async () => {
+		const workbook = new ExcelJS.Workbook()
+		const worksheet = workbook.addWorksheet("Sparse")
+		worksheet.getCell("A1").value = null
+		worksheet.getCell("B1").value = "middle"
+		worksheet.getCell("C1").value = undefined
+		worksheet.getCell("D1").value = "end"
+
+		const expected = "--- Sheet: Sparse ---\n\tmiddle\t\tend"
+		expect(formatWorkbook(workbook)).toBe(expected)
+		const source = Buffer.from(await workbook.xlsx.writeBuffer())
+		expect(await extractTextFromXLSX(source)).toBe(expected)
+	})
+
+	it("decodes only the selected buffer view when another workbook follows in the backing buffer", async () => {
+		const approved = new ExcelJS.Workbook()
+		const worksheet = approved.addWorksheet("Selected")
+		worksheet.getCell("A1").value = "approved cell"
+		worksheet.getCell("B1").value = 42
+		const approvedBytes = Buffer.from(await approved.xlsx.writeBuffer())
+
+		const unapproved = new ExcelJS.Workbook()
+		unapproved.addWorksheet("Unapproved").getCell("A1").value = "outside selected bytes"
+		const unapprovedBytes = Buffer.from(await unapproved.xlsx.writeBuffer())
+		const prefix = Buffer.from("unapproved prefix")
+		const backing = Buffer.concat([prefix, approvedBytes, unapprovedBytes])
+		const source = backing.subarray(prefix.length, prefix.length + approvedBytes.length)
+
+		// Passing source.buffer would expose the later, unapproved ZIP archive to the decoder.
+		expect(source.byteOffset).toBeGreaterThan(0)
+		expect(source.byteLength).toBeLessThan(source.buffer.byteLength)
+		expect(await extractTextFromXLSX(source)).toBe("--- Sheet: Selected ---\napproved cell\t42")
+	})
 
-describe("extractTextFromXLSX", () => {
+	it("rejects invalid workbook bytes instead of returning empty workbook text", async () => {
+		await expect(extractTextFromXLSX(Buffer.from("not an XLSX archive"))).rejects.toThrow()
+	})
+})
+
+describe("formatWorkbook", () => {
 	describe("basic functionality", () => {
-		it("should extract text with proper formatting", async () => {
+		it("should extract text with proper formatting", () => {
 			const workbook = new ExcelJS.Workbook()
 			const worksheet = workbook.addWorksheet("Sheet1")
 
@@ -12,22 +53,22 @@ describe("extractTextFromXLSX", () => {
 			worksheet.getCell("A2").value = "Test"
 			worksheet.getCell("B2").value = 123
 
-			const result = await extractTextFromXLSX(workbook)
+			const result = formatWorkbook(workbook)
 
 			expect(result).toContain("--- Sheet: Sheet1 ---")
 			expect(result).toContain("Hello\tWorld")
 			expect(result).toContain("Test\t123")
 		})
 
-		it("should skip rows with no content", async () => {
+		it("should skip rows with no content", () => {
 			const workbook = new ExcelJS.Workbook()
 			const worksheet = workbook.addWorksheet("Sheet1")
 
 			worksheet.getCell("A1").value = "Row 1"
 			// Row 2 is completely empty
 			worksheet.getCell("A3").value = "Row 3"
 
-			const result = await extractTextFromXLSX(workbook)
+			const result = formatWorkbook(workbook)
 
 			expect(result).toContain("Row 1")
 			expect(result).toContain("Row 3")
@@ -37,7 +78,7 @@ describe("extractTextFromXLSX", () => {
 	})
 
 	describe("sheet handling", () => {
-		it("should process multiple sheets", async () => {
+		it("should process multiple sheets", () => {
 			const workbook = new ExcelJS.Workbook()
 
 			const sheet1 = workbook.addWorksheet("First Sheet")
@@ -46,15 +87,15 @@ describe("extractTextFromXLSX", () => {
 			const sheet2 = workbook.addWorksheet("Second Sheet")
 			sheet2.getCell("A1").value = "Sheet 2 Data"
 
-			const result = await extractTextFromXLSX(workbook)
+			const result = formatWorkbook(workbook)
 
 			expect(result).toContain("--- Sheet: First Sheet ---")
 			expect(result).toContain("Sheet 1 Data")
 			expect(result).toContain("--- Sheet: Second Sheet ---")
 			expect(result).toContain("Sheet 2 Data")
 		})
 
-		it("should skip hidden sheets", async () => {
+		it("should skip hidden sheets", () => {
 			const workbook = new ExcelJS.Workbook()
 
 			const visibleSheet = workbook.addWorksheet("Visible Sheet")
@@ -64,15 +105,15 @@ describe("extractTextFromXLSX", () => {
 			hiddenSheet.getCell("A1").value = "Hidden Data"
 			hiddenSheet.state = "hidden"
 
-			const result = await extractTextFromXLSX(workbook)
+			const result = formatWorkbook(workbook)
 
 			expect(result).toContain("--- Sheet: Visible Sheet ---")
 			expect(result).toContain("Visible Data")
 			expect(result).not.toContain("--- Sheet: Hidden Sheet ---")
 			expect(result).not.toContain("Hidden Data")
 		})
 
-		it("should skip very hidden sheets", async () => {
+		it("should skip very hidden sheets", () => {
 			const workbook = new ExcelJS.Workbook()
 
 			const visibleSheet = workbook.addWorksheet("Visible Sheet")
@@ -82,7 +123,7 @@ describe("extractTextFromXLSX", () => {
 			veryHiddenSheet.getCell("A1").value = "Very Hidden Data"
 			veryHiddenSheet.state = "veryHidden"
 
-			const result = await extractTextFromXLSX(workbook)
+			const result = formatWorkbook(workbook)
 
 			expect(result).toContain("--- Sheet: Visible Sheet ---")
 			expect(result).toContain("Visible Data")
@@ -92,7 +133,7 @@ describe("extractTextFromXLSX", () => {
 	})
 
 	describe("formatCellValue logic", () => {
-		it("should handle null and undefined values", async () => {
+		it("should handle null and undefined values", () => {
 			const workbook = new ExcelJS.Workbook()
 			const worksheet = workbook.addWorksheet("Sheet1")
 
@@ -101,7 +142,7 @@ describe("extractTextFromXLSX", () => {
 			worksheet.getCell("A3").value = undefined
 			worksheet.getCell("A4").value = "After"
 
-			const result = await extractTextFromXLSX(workbook)
+			const result = formatWorkbook(workbook)
 
 			expect(result).toContain("Before")
 			expect(result).toContain("After")
@@ -111,43 +152,43 @@ describe("extractTextFromXLSX", () => {
 			expect(dataLines).toHaveLength(2) // Only 'Before' and 'After' should create content
 		})
 
-		it("should format dates correctly", async () => {
+		it("should format dates correctly", () => {
 			const workbook = new ExcelJS.Workbook()
 			const worksheet = workbook.addWorksheet("Sheet1")
 
 			const testDate = new Date("2023-12-25")
 			worksheet.getCell("A1").value = testDate
 
-			const result = await extractTextFromXLSX(workbook)
+			const result = formatWorkbook(workbook)
 
 			expect(result).toContain("2023-12-25")
 		})
 
-		it("should handle error values", async () => {
+		it("should handle error values", () => {
 			const workbook = new ExcelJS.Workbook()
 			const worksheet = workbook.addWorksheet("Sheet1")
 
 			worksheet.getCell("A1").value = { error: "#DIV/0!" }
 
-			const result = await extractTextFromXLSX(workbook)
+			const result = formatWorkbook(workbook)
 
 			expect(result).toContain("[Error: #DIV/0!]")
 		})
 
-		it("should handle rich text", async () => {
+		it("should handle rich text", () => {
 			const workbook = new ExcelJS.Workbook()
 			const worksheet = workbook.addWorksheet("Sheet1")
 
 			worksheet.getCell("A1").value = {
 				richText: [{ text: "Hello " }, { text: "World", font: { bold: true } }],
 			}
 
-			const result = await extractTextFromXLSX(workbook)
+			const result = formatWorkbook(workbook)
 
 			expect(result).toContain("Hello World")
 		})
 
-		it("should handle hyperlinks", async () => {
+		it("should handle hyperlinks", () => {
 			const workbook = new ExcelJS.Workbook()
 			const worksheet = workbook.addWorksheet("Sheet1")
 
@@ -156,45 +197,58 @@ describe("extractTextFromXLSX", () => {
 				hyperlink: "https://roocode.com/",
 			}
 
-			const result = await extractTextFromXLSX(workbook)
+			const result = formatWorkbook(workbook)
 
 			expect(result).toContain("Roo Code (https://roocode.com/)")
 		})
 
-		it("should handle formulas with and without results", async () => {
+		it("should handle formulas with and without results", () => {
 			const workbook = new ExcelJS.Workbook()
 			const worksheet = workbook.addWorksheet("Sheet1")
 
 			worksheet.getCell("A1").value = { formula: "A2+A3", result: 30 }
 			worksheet.getCell("A2").value = { formula: "SUM(B1:B10)" }
 
-			const result = await extractTextFromXLSX(workbook)
+			const result = formatWorkbook(workbook)
 
 			expect(result).toContain("30") // Formula with result
 			expect(result).toContain("[Formula: SUM(B1:B10)]") // Formula without result
 		})
 	})
 
 	describe("edge cases", () => {
-		it("should handle empty workbook", async () => {
+		it("includes row 50000, marks row 50001 as truncated, and still formats the next sheet", () => {
+			const workbook = new ExcelJS.Workbook()
+			const worksheet = workbook.addWorksheet("Large")
+			worksheet.getCell("A50000").value = "last included row"
+			worksheet.getCell("A50001").value = "excluded row content"
+			workbook.addWorksheet("Next").getCell("A1").value = "next sheet content"
+
+			expect(formatWorkbook(workbook)).toBe(
+				"--- Sheet: Large ---\nlast included row\n[... truncated at row 50001 ...]\n\n" +
+					"--- Sheet: Next ---\nnext sheet content",
+			)
+		})
+
+		it("should handle empty workbook", () => {
 			const workbook = new ExcelJS.Workbook()
 			workbook.addWorksheet("Empty Sheet")
 
-			const result = await extractTextFromXLSX(workbook)
+			const result = formatWorkbook(workbook)
 
 			expect(result).toContain("--- Sheet: Empty Sheet ---")
 			expect(result.trim()).toBe("--- Sheet: Empty Sheet ---")
 		})
 
-		it("should handle workbook with only empty cells", async () => {
+		it("should handle workbook with only empty cells", () => {
 			const workbook = new ExcelJS.Workbook()
 			const worksheet = workbook.addWorksheet("Sheet1")
 
 			// Set cells but leave them empty
 			worksheet.getCell("A1").value = ""
 			worksheet.getCell("B1").value = ""
 
-			const result = await extractTextFromXLSX(workbook)
+			const result = formatWorkbook(workbook)
 
 			expect(result).toContain("--- Sheet: Sheet1 ---")
 			// Should not contain any data rows since empty strings don't count as content
@@ -203,19 +257,19 @@ describe("extractTextFromXLSX", () => {
 		})
 	})
 
-	describe("function overloads", () => {
-		it("should work with workbook objects", async () => {
+	describe("workbook formatting and path adapter", () => {
+		it("should work with workbook objects", () => {
 			const workbook = new ExcelJS.Workbook()
 			const worksheet = workbook.addWorksheet("Test")
 			worksheet.getCell("A1").value = "Test Data"
 
-			const result = await extractTextFromXLSX(workbook)
+			const result = formatWorkbook(workbook)
 
 			expect(result).toContain("Test Data")
 		})
 
 		it("should reject invalid file paths", async () => {
-			await expect(extractTextFromXLSX("/non/existent/file.xlsx")).rejects.toThrow()
+			await expect(extractTextFromFile("/non/existent/file.xlsx")).rejects.toThrow()
 		})
 	})
 })