fix: clip untrusted text without splitting a surrogate pair

apps/server/src/browser.ts

@@ -7,6 +7,7 @@ import type { Config } from "./config.ts";
 import type { Store } from "./db.ts";
 import { AppError } from "./errors.ts";
 import type { Files } from "./files.ts";
+import { clipText } from "./text.ts";
 
 const sessionSchema = z.object({
   id: z.string(),
@@ -220,7 +221,7 @@ export class BrowserService {
       return {
         sessionId: id,
         ...page,
-        text: page.text.slice(0, 30_000),
+        text: clipText(page.text, 30_000),
         truncated: page.truncated || page.text.length > 30_000,
       };
     });

apps/server/src/engine/conversation.ts

@@ -19,13 +19,9 @@ import { JevService } from "../jev/service.ts";
 import { presentChoicesTool } from "../jev/tools.ts";
 import { searchDescription, searchInputSchema, searchInstructions } from "../search.ts";
 import type { AgentService } from "./service.ts";
+import { clipText } from "../text.ts";
 import { tanstackAgent } from "./tanstack-agent.ts";
 
-function clipCalendarText(text: string, limit: number) {
-  const clipped = text.slice(0, limit);
-  return text.length > limit && /[\uD800-\uDBFF]$/.test(clipped) ? clipped.slice(0, -1) : clipped;
-}
-
 export class ConversationAgent extends AbstractAgent {
   constructor(
     private readonly config: Config,
@@ -187,9 +183,9 @@ export class ConversationAgent extends AbstractAgent {
               ...range,
               events: events.map(({ attendees: _, ...event }) => ({
                 ...event,
-                title: clipCalendarText(event.title, 500),
-                location: clipCalendarText(event.location, 500),
-                description: clipCalendarText(event.description, 2000),
+                title: clipText(event.title, 500),
+                location: clipText(event.location, 500),
+                description: clipText(event.description, 2000),
               })),
               truncated:
                 page.truncated ||

apps/server/src/engine/model.ts

@@ -10,6 +10,7 @@ import { searchDescription, searchInputSchema, searchInstructions } from "../sea
 import type { AgentService } from "./service.ts";
 import { tanstackAgent } from "./tanstack-agent.ts";
 import type { TaskContext } from "./worker.ts";
+import { clipText } from "../text.ts";
 
 export type TaskAnswer = { question?: string; answer: string };
 
@@ -186,7 +187,7 @@ export async function executeModelTask(
                 kind: "web",
                 title: source.title ?? source.url,
                 url: source.url,
-                excerpt: source.excerpts.join("\n").slice(0, 500),
+                excerpt: clipText(source.excerpts.join("\n"), 500),
               });
             }
             task = await ctx.checkpoint({ evidence });
@@ -216,11 +217,11 @@ export async function executeModelTask(
                     kind: "web",
                     title: page.title,
                     url: page.url,
-                    excerpt: page.text.slice(0, 500),
+                    excerpt: clipText(page.text, 500),
                   },
                 ],
               });
-              return { ...page, text: page.text.slice(0, 30000) };
+              return { ...page, text: clipText(page.text, 30000) };
             },
           ),
         ]
@@ -400,7 +401,7 @@ export async function executeModelTask(
     });
   });
   if (runError) throw new Error(runError);
-  if (text) await ctx.event("step", "Agent update", text.slice(0, 12000));
+  if (text) await ctx.event("step", "Agent update", clipText(text, 12000));
   return (
     outcome ?? {
       status: "waiting_input",

apps/server/src/engine/service.ts

@@ -43,6 +43,7 @@ import {
   withoutRelativeTimes,
 } from "./page-diff.ts";
 import { LostLeaseError, type TaskContext, TaskWorker } from "./worker.ts";
+import { clipText } from "../text.ts";
 
 const hash = (text: string) => createHash("sha256").update(text).digest("hex");
 /** A change watch's last saved page: its hash, relative-time-free fingerprint and lines. */
@@ -735,7 +736,7 @@ export class AgentService {
     await this.db.insertIfAbsent(owner, "notifications", value);
   }
   mailEvidence(mail: Mail): Evidence {
-    return { id: mail.id, kind: "mail", title: mail.subject, excerpt: mail.body.slice(0, 400) };
+    return { id: mail.id, kind: "mail", title: mail.subject, excerpt: clipText(mail.body, 400) };
   }
   async artifact(
     owner: string,
@@ -1140,7 +1141,7 @@ export class AgentService {
         checks: monitor.checks + 1,
         lastCheckedAt: date(),
         lastHash: currentHash,
-        lastValue: text.slice(0, 1000),
+        lastValue: clipText(text, 1000),
         nextCheckAt,
         error: null,
       },
@@ -1156,11 +1157,11 @@ export class AgentService {
     await ctx.event(
       "observation",
       previousHash ? "Checked for changes" : "Saved the first observation",
-      text.slice(0, 1000),
+      clipText(text, 1000),
     );
     if (shouldNotify) {
       await ctx.guard();
-      await ctx.event("result", "A meaningful change was found", changes || text.slice(0, 500));
+      await ctx.event("result", "A meaningful change was found", changes || clipText(text, 500));
     }
     const count = diff ? countPageDiff(diff) : "";
     return {
@@ -1184,7 +1185,7 @@ export class AgentService {
               title: monitor.title,
               body: changes
                 ? `Changed at ${observation.url}\n${changes}`
-                : `Condition met at ${observation.url}: ${text.slice(0, 240)}`,
+                : `Condition met at ${observation.url}: ${clipText(text, 240)}`,
               key: `monitor:${monitor.id}:${alertSequence}:${currentHash}`,
             }
           : null,
@@ -1196,7 +1197,7 @@ export class AgentService {
           kind: "web",
           title: observation.title,
           url: observation.url,
-          excerpt: text.slice(0, 600),
+          excerpt: clipText(text, 600),
         },
       ],
       plan: task.plan.map((s) => ({ ...s, status: "succeeded" })),

apps/server/src/jev/service.ts

@@ -8,6 +8,7 @@ import {
   jevOptionSchema,
   jevPanelSchema,
 } from "../../../../packages/domain/src/jev.ts";
+import { clipText } from "../text.ts";
 import type { Store } from "../db.ts";
 import { type JevAdapter, rankJevOptions } from "./adapter.ts";
 
@@ -351,7 +352,7 @@ export class JevService {
         runId,
         kind,
         reference,
-        text: text.slice(0, 30_000),
+        text: clipText(text, 30_000),
       });
       return;
     }

apps/server/src/text.ts

@@ -0,0 +1,25 @@
+/**
+ * Clips untrusted text (page text, mail bodies, titles) at a UTF-16 limit
+ * without splitting a surrogate pair: a lone half of a pair is not a
+ * character, renders as a replacement character and breaks consumers that
+ * encode the result to UTF-8. When the cut lands between a pair's high and
+ * low half, the high half is dropped with it.
+ *
+ * Untrusted input may already contain a lone surrogate half — malformed
+ * upstream, a bad copy/paste — that no cut of ours produced. Each lone half
+ * is normalized to U+FFFD before the clip, the same treatment
+ * `String.prototype.toWellFormed()` applies, so the result is always
+ * well-formed UTF-16 and survives a UTF-8 encode even when no cut happens.
+ */
+const LONE_SURROGATE_HALF =
+  /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|[\uDC00-\uDFFF](?<![\uD800-\uDBFF])/gu;
+
+export function clipText(text: string, limit: number): string {
+  const wellFormed = text.replace(LONE_SURROGATE_HALF, "\uFFFD");
+  const clipped = wellFormed.slice(0, limit);
+  return (
+    wellFormed.length > limit && /[\uD800-\uDBFF]$/.test(clipped)
+      ? clipped.slice(0, -1)
+      : clipped
+  );
+}

tests/clip-text.test.ts

@@ -0,0 +1,61 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { clipText } from "../apps/server/src/text.ts";
+
+// The clip sites carry untrusted page and mail text into tool results, evidence
+// and notifications. A UTF-16 cut can land between a surrogate pair's halves;
+// the lone half that survives is not a character (it renders as U+FFFD and
+// breaks consumers that encode the value to UTF-8).
+test("keeps a short string unchanged", () => {
+  assert.equal(clipText("hello", 10), "hello");
+  assert.equal(clipText("hello", 5), "hello");
+});
+
+test("clips a BMP string at the limit", () => {
+  assert.equal(clipText("hello world", 5), "hello");
+});
+
+test("does not split an astral character at the cut", () => {
+  // U+1F44D (thumbs up) is the surrogate pair D83D DC4D in UTF-16.
+  const text = "a".repeat(239) + "\u{1F44D}".repeat(3);
+  const clipped = clipText(text, 240);
+  assert.equal(clipped.length, 239);
+  assert.ok(!/[\uD800-\uDBFF]$/.test(clipped), "must not end on a lone high surrogate");
+  assert.ok(clipped.endsWith("a"));
+});
+
+test("keeps a pair that fits before the limit", () => {
+  const text = "ab\u{1F44D}cd";
+  assert.equal(clipText(text, 4), "ab\u{1F44D}");
+});
+
+test("clips exactly at a pair boundary without dropping a character", () => {
+  const text = "\u{1F44D}\u{1F44E}";
+  assert.equal(clipText(text, 2), "\u{1F44D}");
+  assert.equal(clipText(text, 4), "\u{1F44D}\u{1F44E}");
+});
+
+test("does not trim when the string is exactly the limit", () => {
+  const text = "\u{1F44D}".repeat(6); // 12 UTF-16 units
+  assert.equal(clipText(text, 12), text);
+});
+
+test("normalizes a lone surrogate already present in the input", () => {
+  // "ab" + a lone high surrogate + "cd": the input is malformed without any
+  // cut, so repairing only cut-created halves would let it through.
+  const clipped = clipText("ab\uD800cd", 100);
+  assert.equal(clipped, "ab\uFFFDcd");
+  // the result survives a UTF-8 round trip, e.g. a Postgres insert
+  assert.equal(Buffer.from(clipped, "utf8").toString("utf8"), clipped);
+});
+
+test("normalizes a lone low surrogate in the input", () => {
+  assert.equal(clipText("ab\uDC01cd", 100), "ab\uFFFDcd");
+});
+
+test("normalizes lone halves but keeps a well-formed pair in the same text", () => {
+  // A lone high surrogate followed by another high surrogate, then a valid
+  // emoji pair: only the halves that lack a partner are replaced.
+  const clipped = clipText("a\uD800\uD801b\u{1F44D}c", 100);
+  assert.equal(clipped, "a\uFFFD\uFFFDb\u{1F44D}c");
+});