From ef8dcb215f738e5157cdba5675495b5f09a4b6cc Mon Sep 17 00:00:00 2001 From: collissimo <148455189+collissimo@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:12:46 +0100 Subject: [PATCH 1/2] feat(dictation): dictated prompts carry an [STT] marker Whoever reads a prompt, agents included, can now tell it was spoken and may hold transcription errors. The marker goes in front of the first dictated text of a draft, on desktop, web and iPhone, and stays out of thread titles. Co-Authored-By: Claude Opus 5.5 --- FORK.md | 2 ++ apps/web/src/components/chat/ChatComposer.tsx | 5 ++++- apps/web/src/dictation/useDictation.ts | 5 ++++- packages/client-runtime/package.json | 4 ++++ .../src/operations/threadTitle.test.ts | 11 +++++++++++ .../client-runtime/src/operations/threadTitle.ts | 5 ++++- .../src/voice-input/controller.test.ts | 14 +++++++------- .../client-runtime/src/voice-input/controller.ts | 6 ++++-- .../src/voice-input/dictationMarker.test.ts | 14 ++++++++++++++ .../src/voice-input/dictationMarker.ts | 10 ++++++++++ 10 files changed, 64 insertions(+), 12 deletions(-) create mode 100644 packages/client-runtime/src/operations/threadTitle.test.ts create mode 100644 packages/client-runtime/src/voice-input/dictationMarker.test.ts create mode 100644 packages/client-runtime/src/voice-input/dictationMarker.ts diff --git a/FORK.md b/FORK.md index dfd5da00ef5a8..6f019034b6f77 100644 --- a/FORK.md +++ b/FORK.md @@ -110,6 +110,8 @@ shift-click, or ending with the word "Reinschrift") to get a structured prompt, cancel a recording or to insert the plain transcript while it is being rewritten. Both keys can be rebound in Settings → Keybindings ("Composer: Dictate"). +- Dictated text starts with `[STT]`, once per prompt, so you and agents can tell it was spoken. + Thread titles leave it out. The iPhone dictation does the same; the t3-voice app does not. - The flag under the mic button is the spoken language; the arrows beside it switch it (German, English, French, Spanish, Italian, Dutch, Polish). The choice is kept per client. - The desktop app transcribes on its own computer, also for threads on a remote environment diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index 000f0560c6af9..948f09b162664 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -228,6 +228,7 @@ import { importComposerThreadAttachment } from "./composerThreadImport"; import { observeResize } from "~/lib/observeResize"; const isEnvironmentRequestInvalidError = Schema.is(EnvironmentRequestInvalidError); +import { markDictatedText } from "@t3tools/client-runtime/voice-input/dictationMarker"; import { useDictation } from "../../dictation/useDictation"; import { ComposerDictationButton } from "./ComposerDictationButton"; import { toastManager } from "../ui/toast"; @@ -2474,7 +2475,9 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) // thread may belong to. The desktop app transcribes on this computer either way. transcriptionEnvironmentId: primaryEnvironmentId ?? environmentId, insert: (text) => - insertComposerTextStable(`${text} `, "cursor", { ensureLeadingBoundary: true }), + insertComposerTextStable(`${markDictatedText(promptRef.current, text)} `, "cursor", { + ensureLeadingBoundary: true, + }), }); const toggleDictationStable = useStableCallback(dictation.toggle); const cancelDictationStable = useStableCallback(dictation.cancel); diff --git a/apps/web/src/dictation/useDictation.ts b/apps/web/src/dictation/useDictation.ts index d5e954a354ee0..1f3fe8fef98ca 100644 --- a/apps/web/src/dictation/useDictation.ts +++ b/apps/web/src/dictation/useDictation.ts @@ -1,4 +1,5 @@ import { squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; +import { markDictatedText } from "@t3tools/client-runtime/voice-input/dictationMarker"; import type { DesktopDictationOutcome, DictationAppendInput, @@ -102,6 +103,7 @@ export function useDictation(input: { const unstructuredRef = useRef(null); const inputRef = useRef(input); const languageRef = useRef(language); + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- gives the send timer and async steps the latest props useEffect(() => { inputRef.current = input; languageRef.current = language; @@ -279,7 +281,7 @@ export function useDictation(input: { ); return; } - void navigator.clipboard?.writeText(text); + void navigator.clipboard?.writeText(markDictatedText("", text)); notify( { text: "The composer cannot take text right now · copied to the clipboard", @@ -323,6 +325,7 @@ export function useDictation(input: { return false; }; + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- holds the microphone and timers until unmount useEffect( () => () => { // Leaving the composer (another thread, unmount) ends a recording without inserting it. diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 9600ad7ad356d..6b4b7ba5ecd53 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -131,6 +131,10 @@ "types": "./src/voice-input/index.ts", "default": "./src/voice-input/index.ts" }, + "./voice-input/dictationMarker": { + "types": "./src/voice-input/dictationMarker.ts", + "default": "./src/voice-input/dictationMarker.ts" + }, "./t3ToolSummary": { "types": "./src/t3ToolSummary.ts", "default": "./src/t3ToolSummary.ts" diff --git a/packages/client-runtime/src/operations/threadTitle.test.ts b/packages/client-runtime/src/operations/threadTitle.test.ts new file mode 100644 index 0000000000000..31e642a2b0071 --- /dev/null +++ b/packages/client-runtime/src/operations/threadTitle.test.ts @@ -0,0 +1,11 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { deriveThreadTitleSeed } from "./threadTitle.ts"; + +describe("deriveThreadTitleSeed", () => { + it("leaves the dictation marker out of the title", () => { + expect(deriveThreadTitleSeed({ text: "[STT] Fix the login", attachments: [] })).toBe( + "Fix the login", + ); + }); +}); diff --git a/packages/client-runtime/src/operations/threadTitle.ts b/packages/client-runtime/src/operations/threadTitle.ts index 2fc2da25e4fe3..51fd126c46b16 100644 --- a/packages/client-runtime/src/operations/threadTitle.ts +++ b/packages/client-runtime/src/operations/threadTitle.ts @@ -3,6 +3,8 @@ import { assistantCitationsToPlainText } from "@t3tools/shared/assistantCitation import { replaceComposerContextReferences } from "@t3tools/shared/composerContextReferences"; import { truncate } from "@t3tools/shared/String"; +import { DICTATION_MARKER } from "../voice-input/dictationMarker.ts"; + export interface ThreadTitleSeedInput { readonly text: string; readonly attachments: ReadonlyArray>; @@ -15,7 +17,8 @@ function normalizeTitleSeed(value: string): string { export function threadTitleTextFromPrompt(value: string): string { const imageIds = new Set(); - const prose = replaceComposerContextReferences(value, (reference) => { + const unmarked = value.replace(`${DICTATION_MARKER} `, ""); + const prose = replaceComposerContextReferences(unmarked, (reference) => { if (reference.kind === "image") { imageIds.add(reference.contextId); return ""; diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts index d433748e5bdbb..2068c936fd8c0 100644 --- a/packages/client-runtime/src/voice-input/controller.test.ts +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -86,8 +86,8 @@ describe("resolveTranscriptCommit", () => { expect(resolveTranscriptCommit(captured, captured, "use the mobile skill", "en-US")).toEqual({ kind: "commit", - text: "Fix 🧪 then use the mobile skill please", - selection: { start: tokenStart + "use the mobile skill".length, end: tokenStart + 20 }, + text: "Fix 🧪 then [STT] use the mobile skill please", + selection: { start: tokenStart + 26, end: tokenStart + 26 }, }); }); @@ -201,14 +201,14 @@ describe("VoiceInputController", () => { if (changeDuring === "recording") selected = second; await harness.controller.stop(); - expect(harness.commits.map((commit) => commit.text)).toEqual(["hello first choice"]); + expect(harness.commits.map((commit) => commit.text)).toEqual(["hello [STT] first choice"]); await harness.controller.start(); await harness.controller.stop(); expect(harness.commits.map((commit) => commit.text)).toEqual([ - "hello first choice", - "hello second choice", + "hello [STT] first choice", + "hello [STT] second choice", ]); expect(preparationSignals).toHaveLength(2); expect(transcriptionSignals).toHaveLength(2); @@ -235,7 +235,7 @@ describe("VoiceInputController", () => { await stopping; expect(harness.commits).toEqual([ - { text: "hello new text", selection: { start: 14, end: 14 } }, + { text: "hello [STT] new text", selection: { start: 20, end: 20 } }, ]); expect(harness.deleted).toEqual(["file:///voice.m4a"]); }); @@ -456,7 +456,7 @@ describe("VoiceInputController", () => { await harness.controller.stop(); expect(harness.commits).toEqual([ - { text: "hello new text", selection: { start: 14, end: 14 } }, + { text: "hello [STT] new text", selection: { start: 20, end: 20 } }, ]); expect(harness.controller.currentState.phase).toBe("idle"); }); diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts index b9d0f54f67672..e744406b34e4d 100644 --- a/packages/client-runtime/src/voice-input/controller.ts +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -1,5 +1,6 @@ import { replaceTextRange } from "@t3tools/shared/composerTrigger"; +import { markDictatedText } from "./dictationMarker.ts"; import type { PreparedVoiceTranscription, VoiceTranscriber } from "./transcription.ts"; export const VOICE_RECORDING_LIMIT_SECONDS = 5 * 60; @@ -81,10 +82,11 @@ export function resolveTranscriptCommit( return { kind: "stale" }; } - const replacement = transcript.trim(); - if (replacement.length === 0) { + const trimmedTranscript = transcript.trim(); + if (trimmedTranscript.length === 0) { return { kind: "empty" }; } + const replacement = markDictatedText(captured.text, trimmedTranscript); const isEmptySelection = captured.selection.start === captured.selection.end; const normalizedLocale = locale.replaceAll("_", "-").toLowerCase(); diff --git a/packages/client-runtime/src/voice-input/dictationMarker.test.ts b/packages/client-runtime/src/voice-input/dictationMarker.test.ts new file mode 100644 index 0000000000000..ef916c9c6dd1f --- /dev/null +++ b/packages/client-runtime/src/voice-input/dictationMarker.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { markDictatedText } from "./dictationMarker.ts"; + +describe("markDictatedText", () => { + it("marks the first dictation of a draft", () => { + expect(markDictatedText("", "hello")).toBe("[STT] hello"); + expect(markDictatedText("/review ", "hello")).toBe("[STT] hello"); + }); + + it("leaves later dictations in a marked draft alone", () => { + expect(markDictatedText("[STT] hello ", "again")).toBe("again"); + }); +}); diff --git a/packages/client-runtime/src/voice-input/dictationMarker.ts b/packages/client-runtime/src/voice-input/dictationMarker.ts new file mode 100644 index 0000000000000..c41fb759d7b8f --- /dev/null +++ b/packages/client-runtime/src/voice-input/dictationMarker.ts @@ -0,0 +1,10 @@ +/** Tells whoever reads a prompt, agents included, that it was spoken and may hold transcription errors. */ +export const DICTATION_MARKER = "[STT]"; + +/** + * Marks only the first dictation of a draft. It stays in front of the spoken text rather than at + * the start of the draft, because a typed `/command` only works as the draft's first word. + */ +export function markDictatedText(draft: string, text: string): string { + return draft.includes(DICTATION_MARKER) ? text : `${DICTATION_MARKER} ${text}`; +} From bdd8a41cd721890b23f0ab23758a4fd34ff77486 Mon Sep 17 00:00:00 2001 From: collissimo <148455189+collissimo@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:15:40 +0100 Subject: [PATCH 2/2] test(mobile): voice input expects the dictation marker Co-Authored-By: Claude Opus 5.5 --- .../features/voice-input/voiceInputSession.test.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.test.ts b/apps/mobile/src/features/voice-input/voiceInputSession.test.ts index 9d30e376b54e2..6f00c63d14b6b 100644 --- a/apps/mobile/src/features/voice-input/voiceInputSession.test.ts +++ b/apps/mobile/src/features/voice-input/voiceInputSession.test.ts @@ -147,8 +147,8 @@ describe("global voice input", () => { ); it.each([ - { selection: { start: 6, end: 11 }, expected: "hello spoken text", cursor: 17 }, - { selection: { start: 6, end: 6 }, expected: "hello spoken text world", cursor: 18 }, + { selection: { start: 6, end: 11 }, expected: "hello [STT] spoken text", cursor: 23 }, + { selection: { start: 6, end: 6 }, expected: "hello [STT] spoken text world", cursor: 24 }, ])( "keeps the starting selection $selection after navigation", async ({ selection, expected, cursor }) => { @@ -187,7 +187,7 @@ describe("global voice input", () => { expect(recorder.stop).not.toHaveBeenCalled(); await session.controller.stop(); - expect(drafts.get("first")).toBe("original prompt spoken text"); + expect(drafts.get("first")).toBe("original prompt [STT] spoken text"); expect(drafts.get(visibleDraft)).toBe("other prompt"); expect(session.controller.currentState.phase).toBe("idle"); }); @@ -236,7 +236,7 @@ describe("global voice input", () => { transcription.resolve("spoken text"); await (stopping ?? session.controller.stop()); expect(recorder.record).toHaveBeenCalledTimes(1); - expect(firstCommit).toHaveBeenCalledWith("first spoken text", { start: 17, end: 17 }); + expect(firstCommit).toHaveBeenCalledWith("first [STT] spoken text", { start: 23, end: 23 }); expect(secondCommit).not.toHaveBeenCalled(); }, ); @@ -265,7 +265,7 @@ describe("global voice input", () => { error: null, url: recorder.uri, }); - expect(commit).toHaveBeenCalledWith("first spoken text", { start: 17, end: 17 }); + expect(commit).toHaveBeenCalledWith("first [STT] spoken text", { start: 23, end: 23 }); expect(session.controller.currentState.phase).toBe("idle"); }); @@ -324,7 +324,7 @@ describe("global voice input", () => { expect(recorder.stop).not.toHaveBeenCalled(); expect(session.controller.currentState.phase).toBe("recording"); await session.controller.stop(); - expect(commit).toHaveBeenCalledWith("hello spoken text", { start: 17, end: 17 }); + expect(commit).toHaveBeenCalledWith("hello [STT] spoken text", { start: 23, end: 23 }); }); it("waits for canceled native work before starting a recording for another draft", async () => { @@ -353,7 +353,7 @@ describe("global voice input", () => { await session.controller.stop(); expect(recorder.record).toHaveBeenCalledTimes(1); expect(oldCommit).not.toHaveBeenCalled(); - expect(nextCommit).toHaveBeenCalledWith("second spoken text", { start: 18, end: 18 }); + expect(nextCommit).toHaveBeenCalledWith("second [STT] spoken text", { start: 24, end: 24 }); }); it("holds the screen awake and meters only while recording", async () => {