let agents read email attachments, and fix gmail text read in the wrong charset

packages/api/package.json

@@ -21,10 +21,13 @@
 		"fast-xml-parser": "^5.11.2",
 		"imapflow": "^2.2.6",
 		"mailparser": "^3.9.36",
+		"mammoth": "^1.13.0",
 		"nodemailer": "^10.0.15",
 		"resend": "^6.7.0",
 		"tldts": "^7.4.19",
 		"ts-results-es": "^7.1.0",
+		"unpdf": "^1.9.1",
+		"xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz",
 		"zod": "^4.5.4"
 	},
 	"devDependencies": {

packages/api/src/agent-side/tools.test.ts

@@ -130,7 +130,8 @@ const draft = {id: 'drafts-7-3', values: {folder: 'Draft', from: 'alex@example.c
 const archive = mock();
 const send = mock();
 const sendNew = mock();
-const connector = {list: mock(), get: mock(), create: mock(), update: mock(), remove: mock(), commands: {archive, send}, newCommands: {sendNew}};
+const read = mock();
+const connector = {list: mock(), get: mock(), create: mock(), update: mock(), remove: mock(), commands: {archive, send}, newCommands: {sendNew, read}};
 
 function useConnection(connection: typeof vaultConnection | typeof emailConnection | typeof notionConnection | typeof googleConnection) {
 	loadConnection.mockResolvedValue(Ok({connection, integration: integration(connection.integrationId), connector}));
@@ -228,7 +229,7 @@ describe('listAgentTools', () => {
 		const {listAgentTools} = await import('./tools');
 		const result = await listAgentTools(signedIn, 'mail-1');
 
-		expect(result.unwrap().map((tool) => tool.name)).toEqual(['emails_list', 'emails_get', 'emails_send', 'emails_send_new']);
+		expect(result.unwrap().map((tool) => tool.name)).toEqual(['emails_list', 'emails_get', 'emails_send', 'emails_send_new', 'attachments_read']);
 	});
 });
 
@@ -409,6 +410,18 @@ describe('runAgentTool', () => {
 		expect(logAgentRequest.mock.calls[0]?.[0]).toMatchObject({action: 'sendNew', recordTitle: 'Friday', outcome: 'allowed'});
 	});
 
+	test('an attachment read by its number is logged by the name it came back with', async () => {
+		useConnection(emailConnection);
+		getAgentById.mockResolvedValue(Ok({...agent, grants: [{connectionId: 'mail-1', actionId: 'read', allowed: true}]}));
+		read.mockResolvedValue(Ok({id: 'inbox-7-12/1', values: {email: 'inbox-7-12', attachment: '1', name: 'Budget.xlsx', content: 'Tab: Q3'}, updatedAt: '2026-10-01T00:00:00.000Z'}));
+
+		const {runAgentTool} = await import('./tools');
+		const result = await runAgentTool(caller, 'mail-1', 'attachments_read', {email: 'inbox-7-12', attachment: '1'});
+
+		expect(result.unwrap()).toMatchObject({record: {values: {name: 'Budget.xlsx'}}});
+		expect(logAgentRequest.mock.calls[0]?.[0]).toMatchObject({collectionId: 'attachments', action: 'read', recordTitle: 'Budget.xlsx', outcome: 'allowed'});
+	});
+
 	test('a run the provider fails is logged as failed with what went wrong, not as allowed, and passed on', async () => {
 		useConnection(emailConnection);
 		getAgentById.mockResolvedValue(Ok(triage));

packages/api/src/agent-side/tools.ts

@@ -435,7 +435,8 @@ function newCommand(run: ToolRun, params: Record<string, string>): Promise<Resul
 		const runner = $(requirePresent(run.connector.newCommands?.[run.tool.operation], ApiErr.notFound('tool', run.tool.name)));
 		const values = $(parseRecordValues(run.target.collection, params));
 		const record = $(await logFailure(run, await runner(run.target, values), recordTitle(run.target.collection, values)));
-		$(await log(run, 'allowed', recordTitle(run.target.collection, values)));
+		// Titled as it came back where it did, as an attachment read by its number is by its name.
+		$(await log(run, 'allowed', (record && recordTitle(run.target.collection, record.values)) ?? recordTitle(run.target.collection, values)));
 		return {record};
 	});
 }

packages/api/src/connections/google/gmail-api.test.ts

@@ -9,7 +9,7 @@ mock.module('@proxy/db/connection', () => ({updateConnectionCredential: mock(),
 
 mock.module('../../utils/env', () => ({env: {NODE_ENV: 'production', APP_URL: 'https://app.example.com', GOOGLE_CLIENT_ID: 'client-1', GOOGLE_CLIENT_SECRET: 'secret-1'}}));
 
-const {attachmentNames, folderOf, folderQuery, htmlToText, messageBody} = await import('./gmail-api');
+const {attachmentsOf, folderOf, folderQuery, messageBody} = await import('./gmail-api');
 
 const base64url = (text: string, encoding: BufferEncoding = 'utf8'): string => Buffer.from(text, encoding).toString('base64url');
 
@@ -77,22 +77,45 @@ describe('messageBody', () => {
 		expect(messageBody(textPart('text/html', html))).toBe('Your order has shipped & is on its way.\nTrack it here (https://example.com/track/1).\n\n- One\n- Two');
 	});
 
-	test('nested parts are walked, a text part in the charset it names, and the attachments listed after', () => {
+	test('nested parts are walked, a text part in the charset it names, and the attachments found apart from it', () => {
 		const latin = {mimeType: 'text/plain', filename: '', headers: [{name: 'Content-Type', value: 'text/plain; charset=ISO-8859-1'}], body: {size: 12, data: base64url('Grüße, Sam', 'latin1')}};
 		const payload: MessagePart = {
 			mimeType: 'multipart/mixed',
 			filename: '',
 			parts: [{mimeType: 'multipart/alternative', filename: '', parts: [latin, textPart('text/html', '<p>Grüße</p>')]}, attachment('invoice.pdf'), attachment('receipt.pdf')],
 		};
 
-		expect(messageBody(payload)).toBe('Grüße, Sam\n\nAttachments: invoice.pdf, receipt.pdf');
-		expect(attachmentNames(payload)).toEqual(['invoice.pdf', 'receipt.pdf']);
+		expect(messageBody(payload)).toBe('Grüße, Sam');
+		expect(attachmentsOf(payload)).toEqual([
+			{name: 'invoice.pdf', type: 'application/pdf', size: 1200, data: null, attachmentId: 'att-1'},
+			{name: 'receipt.pdf', type: 'application/pdf', size: 1200, data: null, attachmentId: 'att-1'},
+		]);
+	});
+
+	test('a part naming another charset whose content Gmail hands over as UTF-8 is read as UTF-8', () => {
+		const part = {
+			mimeType: 'text/plain',
+			filename: '',
+			headers: [{name: 'Content-Type', value: 'text/plain; charset="iso-8859-1"'}],
+			body: {size: 30, data: base64url('mit der Zusammenfassung bin ich selbstverständlich einverstanden')},
+		};
+
+		expect(messageBody(part)).toBe('mit der Zusammenfassung bin ich selbstverständlich einverstanden');
+	});
+
+	test('a part in a charset TextDecoder doesn’t know, or naming none, that isn’t UTF-8 is read as Windows-1252', () => {
+		const unknown = {mimeType: 'text/plain', filename: '', headers: [{name: 'Content-Type', value: 'text/plain; charset=x-unknown'}], body: {size: 6, data: base64url('Grüße', 'latin1')}};
+		const unnamed = {mimeType: 'text/plain', filename: '', headers: [{name: 'Content-Type', value: 'text/plain'}], body: {size: 6, data: base64url('Grüße', 'latin1')}};
+
+		expect(messageBody(unknown)).toBe('Grüße');
+		expect(messageBody(unnamed)).toBe('Grüße');
 	});
 
 	test('a text attachment is not the body', () => {
 		const payload: MessagePart = {mimeType: 'multipart/mixed', filename: '', parts: [{...textPart('text/plain', 'notes'), filename: 'notes.txt'}, textPart('text/html', '<p>The body</p>')]};
 
-		expect(messageBody(payload)).toBe('The body\n\nAttachments: notes.txt');
+		expect(messageBody(payload)).toBe('The body');
+		expect(attachmentsOf(payload).map(({name, data}) => [name, data])).toEqual([['notes.txt', base64url('notes')]]);
 	});
 
 	test('a message without parts is empty', () => {
@@ -104,30 +127,3 @@ describe('messageBody', () => {
 		expect(messageBody(textPart('text/html', '<p>Hi\r\nAlex,</p>\r\n<p>See you\nFriday.</p>'))).toBe('Hi Alex,\nSee you Friday.');
 	});
 });
-
-test('htmlToText finds the end of a hidden element after a letter that lowercases to two', () => {
-	expect(htmlToText('<p>İstanbul</p><STYLE>p {}</STYLE><p>after</p>')).toBe('İstanbul\nafter');
-	expect(htmlToText(`<p>${'İ'.repeat(12)} before</p><style>a > b {}</style><p>after</p><p>more</p>`)).toBe(`${'İ'.repeat(12)} before\nafter\nmore`);
-});
-
-test('htmlToText decodes entities only once the tags are gone', () => {
-	expect(htmlToText('<p>&lt;b&gt; is bold &#8212; &#x263A;</p>')).toBe('<b> is bold — ☺');
-});
-
-test('htmlToText reads a number Unicode has no character for as the replacement character', () => {
-	expect(htmlToText('<p>a&#99999999;b&#x110000;c&#0;d</p>')).toBe('a\uFFFDb\uFFFDc\uFFFDd');
-});
-
-test('htmlToText keeps the text of unclosed tags, and of a link that never ends', () => {
-	expect(htmlToText('<p>Hi <b>there<!-- note')).toBe('Hi there');
-	expect(htmlToText('See <a href="https://example.com">this')).toBe('See this');
-	expect(htmlToText('1 < 2 and <script>gone')).toBe('1 < 2 and');
-	expect(htmlToText('<A HREF="https://example.com">Go</A><BR>on<SCRIPT>x</SCRIPT>')).toBe('Go (https://example.com)\non');
-});
-
-test('htmlToText takes time in proportion to the email, however many tags are left open', () => {
-	const started = performance.now();
-	htmlToText(`${'<a href="x">y'.repeat(100_000)}${'<'.repeat(100_000)}${'<!--'.repeat(100_000)}`);
-
-	expect(performance.now() - started).toBeLessThan(1000);
-});

packages/api/src/connections/google/gmail-api.ts

@@ -1,8 +1,10 @@
-import {Ok, Result} from 'ts-results-es';
+import {Ok, type Result} from 'ts-results-es';
 import {z} from 'zod';
 
 import {type ApiError, mapWithLimit} from '@proxy/utils';
 
+import {decodeText} from '../../utils/decode-text';
+import {htmlToText} from '../../utils/html-to-text';
 import {googleDelete, googleRequest, googleWrite} from './google-client';
 
 /**
@@ -132,12 +134,12 @@ export function headerOf(part: MessagePart | undefined, name: string): string {
 	return part?.headers?.find((header) => header.name.toLowerCase() === lower)?.value ?? '';
 }
 
-// A part's content, from base64url, in the charset its Content-Type names; one TextDecoder doesn't
-// know is read as UTF-8.
+// A part's content, from base64url, as UTF-8 where it is, else in the charset its Content-Type
+// names: a part naming ISO-8859-1 can come from Gmail as UTF-8, which read as ISO-8859-1 turns ä
+// into ä.
 function decodePart(part: MessagePart): string {
-	const bytes = Buffer.from(part.body?.data ?? '', 'base64url');
-	const charset = /charset="?([^";\s]+)"?/i.exec(headerOf(part, 'Content-Type'))?.[1] ?? 'utf-8';
-	return Result.wrap(() => new TextDecoder(charset).decode(bytes)).unwrapOr(bytes.toString('utf8'));
+	const charset = /charset="?([^";\s]+)"?/i.exec(headerOf(part, 'Content-Type'))?.[1] ?? null;
+	return decodeText(Buffer.from(part.body?.data ?? '', 'base64url'), charset);
 }
 
 function isAttachment(part: MessagePart): boolean {
@@ -161,134 +163,21 @@ function findText(part: MessagePart, mimeType: string): MessagePart | null {
 	return null;
 }
 
-/** The names of a message's attachments, inline images included, in the order of its parts. */
-export function attachmentNames(part: MessagePart | undefined): string[] {
-	if (!part) {
-		return [];
-	}
-	const own = isAttachment(part) ? [part.filename ?? ''] : [];
-	return [...own, ...(part.parts ?? []).flatMap(attachmentNames)];
-}
-
-const ENTITIES: Record<string, string> = {amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' '};
-
-// A character by its number, or the replacement character for one Unicode doesn't have.
-function codePoint(digits: string, radix: number): string {
-	const value = Number.parseInt(digits, radix);
-	if (!Number.isInteger(value) || value < 1 || value > 0x10ffff) {
-		return '\uFFFD';
-	}
-	return String.fromCodePoint(value);
-}
-
-function decodeEntities(text: string): string {
-	return text.replace(/&(#x[0-9a-f]{1,8}|#\d{1,8}|[a-z]{1,32});/gi, (entity, name: string) => {
-		if (name.startsWith('#x') || name.startsWith('#X')) {
-			return codePoint(name.slice(2), 16);
-		}
-		if (name.startsWith('#')) {
-			return codePoint(name.slice(1), 10);
-		}
-		return ENTITIES[name.toLowerCase()] ?? entity;
-	});
-}
-
-// Elements whose content is never text.
-const HIDDEN = ['script', 'style', 'head', 'title'];
-
-// Elements ending a line where they close.
-const BLOCKS = ['p', 'div', 'tr', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'blockquote', 'table', 'ul', 'ol'];
-
-// What a tag becomes in the text, by its name and whether it closes an element.
-function tagText(name: string, closing: boolean): string {
-	if (name === 'br') {
-		return '\n';
-	}
-	if (name === 'li' && !closing) {
-		return '\n- ';
-	}
-	if (closing && BLOCKS.includes(name)) {
-		return '\n';
-	}
-	return '';
-}
-
 /**
- * HTML as plain text, for an email without a text part: blocks and line breaks as new lines, a
- * link's address after its text where they differ, everything else's tags dropped. Read in one
- * pass, each tag found from where the last ended, so the time it takes grows with the email's
- * length whatever its HTML looks like, unclosed tags and all.
+ * One of a message's attachments: its content is in the message, as a small one's may be, or is
+ * fetched by its `attachmentId`, which Gmail gives anew every time the message is read.
  */
-export function htmlToText(html: string): string {
-	const parts: string[] = [];
-	// The link being read: its address, and where its text starts among the parts.
-	let link: {href: string; start: number} | null = null;
-	let at = 0;
-	while (at < html.length) {
-		const open = html.indexOf('<', at);
-		if (open === -1) {
-			parts.push(sourceText(html.slice(at)));
-			break;
-		}
-		parts.push(sourceText(html.slice(at, open)));
-		// A < that starts no tag, as in "1 < 2", is text.
-		if (!/[a-z/!?]/i.test(html.charAt(open + 1))) {
-			parts.push('<');
-			at = open + 1;
-			continue;
-		}
-		if (html.startsWith('<!--', open)) {
-			const end = html.indexOf('-->', open + 4);
-			at = end === -1 ? html.length : end + 3;
-			continue;
-		}
-		const close = html.indexOf('>', open);
-		if (close === -1) {
-			parts.push(sourceText(html.slice(open)));
-			break;
-		}
-		const tag = html.slice(open, close + 1);
-		at = close + 1;
-		const [, slash = '', rawName = ''] = /^<(\/?)([a-z][a-z0-9]*)/i.exec(tag) ?? [];
-		const name = rawName.toLowerCase();
-		const closing = slash === '/';
-		if (!closing && HIDDEN.includes(name)) {
-			const end = indexOfTag(html, `</${name}`, at);
-			const endClose = end === -1 ? -1 : html.indexOf('>', end);
-			at = endClose === -1 ? html.length : endClose + 1;
-			continue;
-		}
-		if (name === 'a' && !closing) {
-			const href = /\shref\s*=\s*"([^"]*)"/i.exec(tag)?.[1];
-			link = href === undefined ? null : {href, start: parts.length};
-			continue;
-		}
-		if (name === 'a' && closing && link) {
-			const label = parts.splice(link.start).join('').trim();
-			parts.push(label === '' || label === link.href ? link.href : `${label} (${link.href})`);
-			link = null;
-			continue;
-		}
-		parts.push(tagText(name, closing));
-	}
-	return decodeEntities(parts.join(''))
-		.replace(/[ \t\u00a0]+/g, ' ')
-		.replace(/ ?\n ?/g, '\n')
-		.replace(/\n{3,}/g, '\n\n')
-		.trim();
-}
-
-// Text between tags: a line break in the HTML's source is a space, as a browser shows it.
-function sourceText(text: string): string {
-	return text.replace(/\r\n?|\n/g, ' ');
-}
+export type GmailAttachment = {name: string; type: string; size: number; data: string | null; attachmentId: string | null};
 
-// Where a tag next starts from `from`, whatever its case. The email isn't lowercased whole to look,
-// since a letter can lowercase to more than one, as İ does, which would move everything after it.
-function indexOfTag(html: string, tag: string, from: number): number {
-	const pattern = new RegExp(tag, 'ig');
-	pattern.lastIndex = from;
-	return pattern.exec(html)?.index ?? -1;
+/** A message's attachments, inline images included, in the order of its parts. */
+export function attachmentsOf(part: MessagePart | undefined): GmailAttachment[] {
+	if (!part) {
+		return [];
+	}
+	const own = isAttachment(part)
+		? [{name: part.filename ?? '', type: part.mimeType ?? '', size: part.body?.size ?? 0, data: part.body?.data ?? null, attachmentId: part.body?.attachmentId ?? null}]
+		: [];
+	return [...own, ...(part.parts ?? []).flatMap(attachmentsOf)];
 }
 
 /** A message's HTML part, as its sender wrote it, or null for one without. */
@@ -297,19 +186,17 @@ export function messageHtml(payload: MessagePart | undefined): string | null {
 	return html ? decodePart(html) : null;
 }
 
-/** A message's text: its plain text part, else its HTML part as text, then the names of its attachments. */
+/** A message's text: its plain text part, else its HTML part as text. */
 export function messageBody(payload: MessagePart | undefined): string {
 	if (!payload) {
 		return '';
 	}
 	const plain = findText(payload, 'text/plain');
-	const html = plain ? null : findText(payload, 'text/html');
-	const text = plain ? decodePart(plain).replace(/\r\n?/g, '\n').trim() : html ? htmlToText(decodePart(html)) : '';
-	const attachments = attachmentNames(payload);
-	if (attachments.length === 0) {
-		return text;
+	if (plain) {
+		return decodePart(plain).replace(/\r\n?/g, '\n').trim();
 	}
-	return `${text}\n\nAttachments: ${attachments.join(', ')}`.trim();
+	const html = findText(payload, 'text/html');
+	return html ? htmlToText(decodePart(html)) : '';
 }
 
 // ---- Calls -----------------------------------------------------------------------------------
@@ -398,6 +285,14 @@ export function getMessage(accessToken: string, messageId: string): Promise<Resu
 	return googleRequest(accessToken, gmailUrl(`/messages/${messageId}`, {format: 'full'}), messageSchema);
 }
 
+const attachmentSchema = z.object({data: z.string()});
+
+/** An attachment's content, by the id the message gave it when it was read. */
+export async function getAttachment(accessToken: string, messageId: string, attachmentId: string): Promise<Result<Buffer, ApiError>> {
+	const answer = await googleRequest(accessToken, gmailUrl(`/messages/${messageId}/attachments/${attachmentId}`), attachmentSchema);
+	return answer.map(({data}) => Buffer.from(data, 'base64url'));
+}
+
 /** Adds and removes labels on a message, as marking, starring and archiving do. */
 export function modifyMessage(accessToken: string, messageId: string, change: {add?: string[]; remove?: string[]}): Promise<Result<GmailMessage, ApiError>> {
 	return googleWrite(accessToken, 'POST', gmailUrl(`/messages/${messageId}/modify`), {addLabelIds: change.add ?? [], removeLabelIds: change.remove ?? []}, messageSchema);