From 88de7f243035e5a79b0a47f8cc3d55e54526b68a Mon Sep 17 00:00:00 2001 From: collissimo <148455189+collissimo@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:39:39 +0200 Subject: [PATCH 1/5] feat(server): dictation with a local Parakeet model The server turns streamed microphone audio into text with Parakeet through transcribe.cpp, on its own machine. The model (found in the Scrawl or Hex model folders, or T3CODE_DICTATION_MODEL) loads on the first chunk while the user already speaks and is let go after 15 minutes unused. One model serves every client. - dictation.append/finish/cancel: a recording session with a live preview of the last 20 seconds. - Filler sounds are removed; a closing "Reinschrift" asks for a structured rewrite. - dictation.structure: rewrites a rambling transcript into a structured prompt with Claude Haiku (claude --print). Co-Authored-By: Claude Opus 5.5 --- apps/server/package.json | 1 + apps/server/src/auth/RpcAuthorization.ts | 5 + apps/server/src/dictation/Dictation.ts | 170 ++++++++++ .../src/dictation/DictationEngine.test.ts | 179 ++++++++++ apps/server/src/dictation/DictationEngine.ts | 314 ++++++++++++++++++ apps/server/src/dictation/modelFiles.ts | 92 +++++ apps/server/src/dictation/structurePrompt.ts | 114 +++++++ .../src/dictation/transcriptText.test.ts | 68 ++++ apps/server/src/dictation/transcriptText.ts | 81 +++++ .../src/observability/RpcInstrumentation.ts | 4 + apps/server/src/server.ts | 2 + apps/server/src/ws.test.ts | 2 + apps/server/src/ws.ts | 9 + packages/contracts/src/dictation.ts | 70 ++++ packages/contracts/src/index.ts | 1 + packages/contracts/src/rpc.ts | 42 +++ pnpm-lock.yaml | 269 ++++++++++++++- pnpm-workspace.yaml | 2 + 18 files changed, 1413 insertions(+), 12 deletions(-) create mode 100644 apps/server/src/dictation/Dictation.ts create mode 100644 apps/server/src/dictation/DictationEngine.test.ts create mode 100644 apps/server/src/dictation/DictationEngine.ts create mode 100644 apps/server/src/dictation/modelFiles.ts create mode 100644 apps/server/src/dictation/structurePrompt.ts create mode 100644 apps/server/src/dictation/transcriptText.test.ts create mode 100644 apps/server/src/dictation/transcriptText.ts create mode 100644 packages/contracts/src/dictation.ts diff --git a/apps/server/package.json b/apps/server/package.json index b13957b82b7d2..1fa8eb8e159d9 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -61,6 +61,7 @@ "proper-lockfile": "4.1.2", "stream-chain": "^4.2.5", "stream-json": "3.6.0", + "transcribe-cpp": "0.4.0", "undici": "catalog:", "yaml": "catalog:", "yauzl": "^3.4.0" diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index 18919cea55ecf..53e85c9a8a479 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -84,6 +84,11 @@ export const RPC_REQUIRED_SCOPES = { [WS_METHODS.serverGetDatabaseStorage]: AuthOrchestrationReadScope, [WS_METHODS.serverGetSettings]: AuthOrchestrationReadScope, [WS_METHODS.serverUpdateSettings]: AuthSettingsWriteScope, + // Dictation is input to a prompt, and a structured rewrite runs a model on this environment. + [WS_METHODS.dictationAppend]: AuthOrchestrationOperateScope, + [WS_METHODS.dictationFinish]: AuthOrchestrationOperateScope, + [WS_METHODS.dictationCancel]: AuthOrchestrationOperateScope, + [WS_METHODS.dictationStructure]: AuthOrchestrationOperateScope, [WS_METHODS.serverSearchAcpRegistry]: AuthOrchestrationReadScope, [WS_METHODS.serverPrepareAcpRegistryAgent]: AuthProvidersManageScope, [WS_METHODS.serverUninstallAcpRegistryManagedBinary]: AuthProvidersManageScope, diff --git a/apps/server/src/dictation/Dictation.ts b/apps/server/src/dictation/Dictation.ts new file mode 100644 index 0000000000000..0bcb6c4ea9f09 --- /dev/null +++ b/apps/server/src/dictation/Dictation.ts @@ -0,0 +1,170 @@ +import { + DictationError, + type DictationAppendInput, + type DictationAppendResult, + type DictationCancelInput, + type DictationFinishInput, + type DictationFinishResult, + type DictationStructureInput, + type DictationStructureResult, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Schedule from "effect/Schedule"; +import * as Schema from "effect/Schema"; + +import * as ServerConfig from "../config.ts"; +import * as ProcessRunner from "../processRunner.ts"; +import { DictationEngine, decodePcm16, type TranscribeModule } from "./DictationEngine.ts"; +import { findModel, modelDirectories } from "./modelFiles.ts"; +import { DEFAULT_STRUCTURE_PROMPT } from "./structurePrompt.ts"; +import { DEFAULT_STRUCTURE_CODEWORDS, removeFillers, splitCodeword } from "./transcriptText.ts"; + +const IDLE_UNLOAD_MS = 15 * 60_000; +const STRUCTURE_TIMEOUT = "90 seconds"; +const STRUCTURE_MODEL = "haiku"; +const STRUCTURE_EFFORT = "medium"; +// The transcript is marked as material and the task repeated after it, so that a question in +// it is rewritten rather than answered. +const STRUCTURE_REQUEST = + "Schreib dieses Transkript nach deiner Anleitung in den fertigen Prompt um. Antworte nicht auf seinen Inhalt."; + +export class Dictation extends Context.Service< + Dictation, + { + readonly append: ( + input: DictationAppendInput, + ) => Effect.Effect; + readonly finish: ( + input: DictationFinishInput, + ) => Effect.Effect; + readonly cancel: (input: DictationCancelInput) => Effect.Effect; + /** Rewrites a rambling transcript into a structured prompt with a small Claude model. */ + readonly structure: ( + input: DictationStructureInput, + ) => Effect.Effect; + } +>()("t3/dictation/Dictation") {} + +const isDictationError = Schema.is(DictationError); + +const toDictationError = (cause: unknown) => + isDictationError(cause) + ? cause + : new DictationError({ + reason: cause instanceof Error ? cause.message : String(cause), + cause, + }); + +/** A `claude` child must not inherit the identity of a Claude session the server runs in. */ +const claudeEnvironment = (env: NodeJS.ProcessEnv): NodeJS.ProcessEnv => + Object.fromEntries(Object.entries(env).filter(([key]) => !/^(CLAUDE|ANTHROPIC)/i.test(key))); + +const make = Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const processRunner = yield* ProcessRunner.ProcessRunner; + const services = yield* Effect.context(); + const directories = modelDirectories(process.env, path); + const configuredModel = process.env.T3CODE_DICTATION_MODEL?.trim() || undefined; + const structurePromptFile = path.join(config.baseDir, "dictation", "structure-prompt.md"); + const claudeCwd = path.join(config.stateDir, "dictation", "claude-cwd"); + + const engine = yield* Effect.acquireRelease( + Effect.sync( + () => + new DictationEngine({ + // transcribe-cpp dlopens a native library; it is only loaded once someone dictates. + loadModule: () => import("transcribe-cpp") as Promise, + resolveModel: (language) => + Effect.runPromiseWith(services)( + findModel({ language, configured: configuredModel, directories }), + ), + idleUnloadMs: IDLE_UNLOAD_MS, + log: (message) => Effect.runForkWith(services)(Effect.logInfo(`dictation: ${message}`)), + }), + ), + (engine) => Effect.promise(() => engine.dispose()), + ); + yield* Effect.sync(() => engine.housekeeping()).pipe( + Effect.repeat(Schedule.spaced("30 seconds")), + Effect.forkScoped, + ); + + const append = Effect.fn("Dictation.append")(function* (input: DictationAppendInput) { + const preview = yield* Effect.try({ + try: () => engine.append(input.sessionId, input.language, decodePcm16(input.pcm)), + catch: toDictationError, + }); + return { preview }; + }); + + const finish = Effect.fn("Dictation.finish")(function* (input: DictationFinishInput) { + const result = yield* Effect.tryPromise({ + try: () => engine.finish(input.sessionId), + catch: toDictationError, + }); + if (result.reason !== undefined) { + return { text: "", reason: result.reason, structureRequested: false }; + } + const cleaned = splitCodeword(removeFillers(result.text), DEFAULT_STRUCTURE_CODEWORDS); + if (cleaned.text === "") { + return { text: "", reason: "no-speech" as const, structureRequested: false }; + } + return { text: cleaned.text, structureRequested: cleaned.found }; + }); + + const cancel = (input: DictationCancelInput) => Effect.sync(() => engine.cancel(input.sessionId)); + + const structure = Effect.fn("Dictation.structure")(function* (input: DictationStructureInput) { + const custom = yield* fs + .readFileString(structurePromptFile) + .pipe(Effect.orElseSucceed(() => "")); + const instructions = custom.trim() || DEFAULT_STRUCTURE_PROMPT; + yield* fs.makeDirectory(claudeCwd, { recursive: true }).pipe(Effect.ignore); + // Safe mode keeps the user's CLAUDE.md, plugins and hooks out: they are written for coding + // sessions and would bend the rewrite. No tools, no MCP servers, nothing saved to disk. + const output = yield* processRunner + .run({ + command: "claude", + args: [ + "--print", + "--model", + STRUCTURE_MODEL, + "--effort", + STRUCTURE_EFFORT, + "--system-prompt", + instructions, + "--safe-mode", + "--strict-mcp-config", + "--no-session-persistence", + "--tools", + "", + ], + cwd: claudeCwd, + env: claudeEnvironment(process.env), + stdin: `\n${input.text}\n\n\n${STRUCTURE_REQUEST}`, + timeout: STRUCTURE_TIMEOUT, + }) + .pipe(Effect.mapError(toDictationError)); + const text = output.stdout.trim(); + if (output.code !== 0 || text === "") { + const detail = (output.stderr.trim() || text || "no output").slice(-400); + return yield* new DictationError({ + reason: `claude exited with ${String(output.code)}: ${detail}`, + }); + } + yield* Effect.logInfo( + `dictation: structured ${input.text.length} characters into ${text.length}`, + ); + return { text }; + }); + + return Dictation.of({ append, finish, cancel, structure }); +}); + +export const layer = Layer.effect(Dictation, make).pipe(Layer.provide(ProcessRunner.layer)); diff --git a/apps/server/src/dictation/DictationEngine.test.ts b/apps/server/src/dictation/DictationEngine.test.ts new file mode 100644 index 0000000000000..1759f7ac4efaf --- /dev/null +++ b/apps/server/src/dictation/DictationEngine.test.ts @@ -0,0 +1,179 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +import { + DictationEngine, + SAMPLE_RATE, + decodePcm16, + normalizeLanguage, + type TranscribeModule, +} from "./DictationEngine.ts"; +import { findModel, rankModels } from "./modelFiles.ts"; + +const V3 = "C:/models/parakeet-tdt-0.6b-v3-Q8_0.gguf"; +const UNIFIED_EN = "C:/models/parakeet-unified-en-0.6b-Q8_0.gguf"; +const V2 = "C:/models/parakeet-tdt-0.6b-v2-Q8_0.gguf"; +const WHISPER = "C:/models/whisper-large-v3-turbo-Q8_0.gguf"; +const basename = (path: string) => path.split("/").at(-1)!; + +describe("rankModels", () => { + it("gives German a multilingual model, Parakeet first", () => { + expect(rankModels([WHISPER, UNIFIED_EN, V2, V3], "de", basename)).toEqual([V3, WHISPER]); + }); + it("prefers Parakeet Unified English for English", () => { + expect(rankModels([V3, WHISPER, UNIFIED_EN], "en", basename)).toEqual([ + UNIFIED_EN, + V3, + WHISPER, + ]); + }); + it("never gives German an English-only model", () => { + expect(rankModels([UNIFIED_EN, V2], "de", basename)).toEqual([]); + }); +}); + +describe("findModel", () => { + it.effect("looks in every folder and skips other files", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const hex = yield* fs.makeTempDirectoryScoped(); + const scrawl = yield* fs.makeTempDirectoryScoped(); + for (const [folder, name] of [ + [hex, "parakeet-unified-en-0.6b-Q8_0.gguf"], + [scrawl, "parakeet-tdt-0.6b-v3-Q8_0.gguf"], + [scrawl, "parakeet-tdt-0.6b-v3-Q8_0.gguf.verified.json"], + [scrawl, "parakeet-tdt-0.6b-v3-Q8_0.gguf.partial"], + ] as const) { + yield* fs.writeFileString(path.join(folder, name), ""); + } + const directories = [hex, path.join(hex, "missing"), scrawl]; + expect(yield* findModel({ language: "de", configured: undefined, directories })).toBe( + path.join(scrawl, "parakeet-tdt-0.6b-v3-Q8_0.gguf"), + ); + expect(yield* findModel({ language: "en", configured: undefined, directories })).toBe( + path.join(hex, "parakeet-unified-en-0.6b-Q8_0.gguf"), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("names the folders when nothing is installed", () => + Effect.gen(function* () { + const error = yield* findModel({ + language: "de", + configured: undefined, + directories: ["C:/nowhere"], + }).pipe(Effect.flip); + expect(error.reason).toContain("C:/nowhere"); + }).pipe(Effect.provide(NodeServices.layer)), + ); +}); + +describe("audio input", () => { + it("decodes 16-bit PCM", () => { + const pcm = Buffer.alloc(4); + pcm.writeInt16LE(16384, 0); + pcm.writeInt16LE(-32768, 2); + expect([...decodePcm16(pcm.toString("base64"))]).toEqual([0.5, -1]); + }); + it("normalizes language tags", () => { + expect(normalizeLanguage("de-DE")).toBe("de"); + expect(normalizeLanguage(" EN_us ")).toBe("en"); + expect(normalizeLanguage("")).toBeUndefined(); + expect(normalizeLanguage(undefined)).toBeUndefined(); + }); +}); + +class FakeModel { + static loads: Array = []; + runs: Array<{ length: number; language: string | undefined }> = []; + disposed = false; + readonly backend = "CPU"; + readonly capabilities = { languages: ["de", "en"] }; + static async load(path: string) { + FakeModel.loads.push(path); + return new FakeModel(); + } + createSession() { + return { + run: async (pcm: Float32Array, opts: { language?: string }) => { + this.runs.push({ length: pcm.length, language: opts.language }); + return { text: " Hallo Welt ", aborted: false }; + }, + dispose: () => undefined, + }; + } + dispose() { + this.disposed = true; + } +} + +const fakeModule = { TranscribeModel: FakeModel, InputTooShort: class extends Error {} }; + +const makeEngine = (resolveModel = async () => "model.gguf") => { + let now = 0; + const engine = new DictationEngine({ + loadModule: async () => fakeModule as unknown as TranscribeModule, + resolveModel, + idleUnloadMs: 60_000, + now: () => now, + }); + return { engine, advance: (ms: number) => (now += ms) }; +}; + +const speech = (seconds: number) => new Float32Array(seconds * SAMPLE_RATE).fill(0.25); + +describe("DictationEngine", () => { + it("records while the model loads and transcribes the whole recording", async () => { + FakeModel.loads = []; + const { engine } = makeEngine(); + engine.append("a", "de-DE", speech(1)); + engine.append("a", "de-DE", speech(2)); + expect(await engine.finish("a")).toEqual({ text: "Hallo Welt" }); + expect(FakeModel.loads).toEqual(["model.gguf"]); + }); + + it("reports silence and too short recordings without asking the model", async () => { + const { engine } = makeEngine(); + engine.append("quiet", "de", new Float32Array(SAMPLE_RATE)); + expect(await engine.finish("quiet")).toEqual({ text: "", reason: "no-audio" }); + engine.append("short", "de", speech(0.1)); + expect(await engine.finish("short")).toEqual({ text: "", reason: "no-speech" }); + expect(await engine.finish("never-started")).toEqual({ text: "", reason: "no-speech" }); + }); + + it("fails the next append when no model can be loaded, and tries again on a new recording", async () => { + let attempts = 0; + const { engine } = makeEngine(async () => { + attempts++; + throw new Error("No Parakeet model found."); + }); + engine.append("a", "de", speech(1)); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(() => engine.append("a", "de", speech(1))).toThrow("No Parakeet model found."); + // A new recording starts over: the model may have been installed meanwhile. + expect(engine.append("b", "de", speech(1))).toBe(""); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(attempts).toBe(2); + expect(() => engine.append("b", "de", speech(1))).toThrow("No Parakeet model found."); + }); + + it("lets go of the model after the idle time, never during a recording", async () => { + const { engine, advance } = makeEngine(); + engine.append("a", "de", speech(1)); + await engine.finish("a"); + expect(engine.modelLoaded).toBe(true); + engine.append("b", "de", speech(1)); + advance(61_000); + engine.housekeeping(); + expect(engine.modelLoaded).toBe(true); + await engine.finish("b"); + advance(61_000); + engine.housekeeping(); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(engine.modelLoaded).toBe(false); + }); +}); diff --git a/apps/server/src/dictation/DictationEngine.ts b/apps/server/src/dictation/DictationEngine.ts new file mode 100644 index 0000000000000..e8254e91eebb1 --- /dev/null +++ b/apps/server/src/dictation/DictationEngine.ts @@ -0,0 +1,314 @@ +// Speech to text on the server's machine: Parakeet through transcribe.cpp. +// +// A client streams microphone audio into a recording session. The model loads on the first +// chunk, while the user is already speaking, and is let go after a stretch without +// recordings (it holds about 1.1 GB). While recording, the last seconds are transcribed again +// and again for a live preview; the final transcript is one run over the whole recording. + +import type * as TranscribeCpp from "transcribe-cpp"; + +export const SAMPLE_RATE = 16_000; +// Below this the model refuses the input anyway. +const MIN_SECONDS = 0.3; +// Peak level under which the microphone delivered nothing usable. +const SILENT_PEAK = 0.002; +// The preview transcribes only the tail of the recording, so its cost does not grow with the +// length. The client shows the last few hundred characters, which hides the clipped first word. +const PREVIEW_WINDOW_SECONDS = 20; +const PREVIEW_PAUSE_MS = 800; +const MAX_RECORDING_SECONDS = 15 * 60; +// A client that vanished mid-recording leaves its session behind. +const ABANDONED_SESSION_MS = 2 * 60_000; + +export type TranscribeModule = typeof TranscribeCpp; + +export interface DictationEngineOptions { + readonly loadModule: () => Promise; + /** Path of the model file that suits the language, or a rejection saying why there is none. */ + readonly resolveModel: (language: string | undefined) => Promise; + /** Let go of the model after this long without a recording; 0 keeps it. */ + readonly idleUnloadMs: number; + /** transcribe.cpp backend. The CPU is the default: Vulkan returns empty text on some GPUs. */ + readonly backend?: TranscribeCpp.Backend; + readonly now?: () => number; + readonly log?: (message: string) => void; +} + +export type TranscriptResult = + | { readonly text: string; readonly reason?: undefined } + | { readonly text: ""; readonly reason: "no-speech" | "no-audio" }; + +interface LoadedModel { + readonly path: string; + readonly model: TranscribeCpp.TranscribeModel; + readonly session: TranscribeCpp.Session; + readonly languages: ReadonlyArray; +} + +interface RecordingSession { + samples: Float32Array; + length: number; + readonly language: string | undefined; + preview: string; + lastActiveAt: number; + lastPreviewAt: number; +} + +/** "de-DE" -> "de"; the model knows languages by their two-letter code. */ +export function normalizeLanguage(language: string | undefined): string | undefined { + const code = language?.trim().toLowerCase().split(/[-_]/)[0]; + return code ? code : undefined; +} + +export function decodePcm16(base64: string): Float32Array { + const bytes = Buffer.from(base64, "base64"); + const count = Math.floor(bytes.length / 2); + const samples = new Float32Array(count); + for (let index = 0; index < count; index++) { + samples[index] = bytes.readInt16LE(index * 2) / 32768; + } + return samples; +} + +export class DictationEngine { + readonly #options: DictationEngineOptions; + readonly #now: () => number; + readonly #sessions = new Map(); + #module: Promise | undefined; + #loaded: LoadedModel | undefined; + #loading: Promise | undefined; + #loadFailure: Error | undefined; + // One native session serves every run, one at a time. + #busy: Promise = Promise.resolve(); + #previewAbort: AbortController | undefined; + #previewRun: Promise | undefined; + #lastUsedAt: number; + + constructor(options: DictationEngineOptions) { + this.#options = options; + this.#now = options.now ?? Date.now; + this.#lastUsedAt = this.#now(); + } + + /** Adds recorded audio to a session, opening it on the first call. Returns the live preview. */ + append(sessionId: string, language: string | undefined, samples: Float32Array): string { + let session = this.#sessions.get(sessionId); + if (session === undefined) { + if ( + this.#loadFailure !== undefined && + this.#loaded === undefined && + this.#loading === undefined + ) { + // A new recording tries again: the model may have been installed meanwhile. + this.#loadFailure = undefined; + } + session = { + samples: new Float32Array(SAMPLE_RATE * 10), + length: 0, + language: normalizeLanguage(language), + preview: "", + lastActiveAt: this.#now(), + lastPreviewAt: this.#now(), + }; + this.#sessions.set(sessionId, session); + void this.#ensureModel(session.language).catch(() => undefined); + } + if (this.#loadFailure !== undefined) { + this.#sessions.delete(sessionId); + throw this.#loadFailure; + } + this.#push(session, samples); + session.lastActiveAt = this.#now(); + this.#maybePreview(session); + return session.preview; + } + + /** Ends a session and transcribes all of it. */ + async finish(sessionId: string): Promise { + const session = this.#sessions.get(sessionId); + this.#sessions.delete(sessionId); + this.#lastUsedAt = this.#now(); + if (session === undefined || session.length < MIN_SECONDS * SAMPLE_RATE) { + return { text: "", reason: "no-speech" }; + } + const audio = session.samples.subarray(0, session.length); + let peak = 0; + for (const sample of audio) peak = Math.max(peak, Math.abs(sample)); + if (peak < SILENT_PEAK) return { text: "", reason: "no-audio" }; + const loaded = await this.#ensureModel(session.language); + // A preview run in flight is aborted instead of making the final run wait for it. + this.#previewAbort?.abort(); + const started = this.#now(); + const text = await this.#exclusive(async () => { + try { + const result = await loaded.session.run(audio, this.#runOptions(loaded, session.language)); + return result.text; + } catch (error) { + const module = await this.#loadModule(); + if (error instanceof module.InputTooShort) return ""; + throw error; + } + }); + this.#lastUsedAt = this.#now(); + const cleaned = text.split(/\s+/).filter(Boolean).join(" "); + this.#options.log?.( + `transcribed ${(audio.length / SAMPLE_RATE).toFixed(1)}s of audio in ${this.#now() - started}ms`, + ); + return cleaned ? { text: cleaned } : { text: "", reason: "no-speech" }; + } + + cancel(sessionId: string): void { + this.#sessions.delete(sessionId); + if (this.#sessions.size === 0) this.#previewAbort?.abort(); + } + + get modelLoaded(): boolean { + return this.#loaded !== undefined; + } + + async dispose(): Promise { + this.#sessions.clear(); + this.#previewAbort?.abort(); + await this.#unload(); + } + + #push(session: RecordingSession, samples: Float32Array): void { + const room = MAX_RECORDING_SECONDS * SAMPLE_RATE - session.length; + const incoming = samples.length > room ? samples.subarray(0, Math.max(0, room)) : samples; + if (session.length + incoming.length > session.samples.length) { + const grown = new Float32Array( + Math.max(session.samples.length * 2, session.length + incoming.length), + ); + grown.set(session.samples.subarray(0, session.length)); + session.samples = grown; + } + session.samples.set(incoming, session.length); + session.length += incoming.length; + } + + #maybePreview(session: RecordingSession): void { + const loaded = this.#loaded; + if ( + loaded === undefined || + this.#previewRun !== undefined || + session.length < SAMPLE_RATE || + this.#now() - session.lastPreviewAt < PREVIEW_PAUSE_MS + ) { + return; + } + session.lastPreviewAt = this.#now(); + const start = Math.max(0, session.length - PREVIEW_WINDOW_SECONDS * SAMPLE_RATE); + // A copy: the recording buffer is replaced when it grows, and native code reads this one. + const tail = session.samples.slice(start, session.length); + const abort = new AbortController(); + this.#previewAbort = abort; + const run = this.#exclusive(() => + abort.signal.aborted + ? Promise.resolve(undefined) + : loaded.session.run(tail, { + ...this.#runOptions(loaded, session.language), + signal: abort.signal, + }), + ) + .then((result) => { + if (result !== undefined && !abort.signal.aborted && !result.aborted) { + session.preview = result.text.split(/\s+/).filter(Boolean).join(" "); + } + }) + // Aborted when the recording ends; anything else only costs this preview. + .catch(() => undefined) + .finally(() => { + if (this.#previewRun === run) this.#previewRun = undefined; + if (this.#previewAbort === abort) this.#previewAbort = undefined; + }); + this.#previewRun = run; + } + + #runOptions(loaded: LoadedModel, language: string | undefined): TranscribeCpp.TranscribeOptions { + // A model that cannot be told the language detects it. + return language !== undefined && loaded.languages.includes(language) ? { language } : {}; + } + + #exclusive(task: () => Promise): Promise { + const run = this.#busy.then(task, task); + this.#busy = run.catch(() => undefined); + return run; + } + + #loadModule(): Promise { + this.#module ??= this.#options.loadModule().catch((error: unknown) => { + this.#module = undefined; + throw new Error(`transcribe.cpp cannot be loaded: ${describe(error)}`, { cause: error }); + }); + return this.#module; + } + + async #ensureModel(language: string | undefined): Promise { + if (this.#loading !== undefined) return this.#loading; + const loading = (async () => { + const path = await this.#options.resolveModel(language); + const current = this.#loaded; + if (current !== undefined && (current.path === path || this.#sessions.size > 1)) { + // Another recording may still need the loaded model; it serves any language it can. + return current; + } + await this.#unload(); + const module = await this.#loadModule(); + const started = this.#now(); + const model = await module.TranscribeModel.load(path, { + backend: this.#options.backend ?? "cpu", + }); + const loaded: LoadedModel = { + path, + model, + session: model.createSession(), + languages: model.capabilities.languages, + }; + this.#loaded = loaded; + this.#lastUsedAt = this.#now(); + this.#options.log?.(`loaded ${path} on ${model.backend} in ${this.#now() - started}ms`); + return loaded; + })(); + this.#loading = loading; + try { + return await loading; + } catch (error) { + this.#loadFailure = error instanceof Error ? error : new Error(describe(error)); + throw this.#loadFailure; + } finally { + this.#loading = undefined; + } + } + + async #unload(): Promise { + const loaded = this.#loaded; + if (loaded === undefined) return; + this.#loaded = undefined; + await this.#busy; + loaded.session.dispose(); + loaded.model.dispose(); + } + + /** Drops abandoned recordings and lets go of an unused model; called every half minute. */ + housekeeping(): void { + const now = this.#now(); + for (const [id, session] of this.#sessions) { + if (now - session.lastActiveAt > ABANDONED_SESSION_MS) this.#sessions.delete(id); + } + const limit = this.#options.idleUnloadMs; + if ( + limit > 0 && + this.#loaded !== undefined && + this.#loading === undefined && + this.#sessions.size === 0 && + now - this.#lastUsedAt > limit + ) { + this.#options.log?.("unused, letting go of the speech model"); + void this.#unload(); + } + } +} + +function describe(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/apps/server/src/dictation/modelFiles.ts b/apps/server/src/dictation/modelFiles.ts new file mode 100644 index 0000000000000..f70c9a95bb441 --- /dev/null +++ b/apps/server/src/dictation/modelFiles.ts @@ -0,0 +1,92 @@ +import { DictationError } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +// The speech model is never downloaded here. Hex (the Windows build of voice-control) and +// Scrawl install Parakeet as a .gguf file; dictation picks it up from their folders. +export function modelDirectories(env: NodeJS.ProcessEnv, path: Path.Path): ReadonlyArray { + const dirs: Array = []; + if (env.APPDATA) dirs.push(path.join(env.APPDATA, "voice-control", "models")); + if (env.LOCALAPPDATA) dirs.push(path.join(env.LOCALAPPDATA, "Scrawl", "models")); + return dirs; +} + +const nameParts = (name: string) => name.toLowerCase().replace(/[_.]/g, "-").split("-"); +const isEnglishOnly = (name: string) => { + const parts = nameParts(name); + return parts.includes("en") || parts.includes("v2"); +}; +const isMultilingual = (name: string) => { + const lowered = name.toLowerCase(); + return !isEnglishOnly(name) && (lowered.includes("v3") || lowered.includes("whisper")); +}; + +/** + * The models that can transcribe `language`, best first. English prefers Parakeet Unified + * English, then any Parakeet. Every other language needs a model whose name says it is + * multilingual (Parakeet v3 covers 25 European languages, or Whisper); Parakeet Unified English + * and Parakeet v2 never get German speech. Without a language, multilingual models come first. + */ +export function rankModels( + paths: ReadonlyArray, + language: string | undefined, + basename: (path: string) => string, +): ReadonlyArray { + const english = language?.toLowerCase().startsWith("en") ?? false; + const rank = (path: string): ReadonlyArray => { + const name = basename(path).toLowerCase(); + return [ + !name.startsWith("parakeet"), + !(english && name.includes("unified-en")), + !(!english && name.includes("v3")), + name, + ]; + }; + const compare = (left: string, right: string) => { + const a = rank(left); + const b = rank(right); + for (let index = 0; index < a.length; index++) { + if (a[index]! < b[index]!) return -1; + if (a[index]! > b[index]!) return 1; + } + return 0; + }; + return paths.filter((path) => english || isMultilingual(basename(path))).toSorted(compare); +} + +/** The model file to load: `configured` when set, else the best one Hex or Scrawl installed. */ +export const findModel = Effect.fn("dictation.findModel")(function* (input: { + readonly language: string | undefined; + readonly configured: string | undefined; + readonly directories: ReadonlyArray; +}) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + if (input.configured) { + const exists = yield* fs.exists(input.configured).pipe(Effect.orElseSucceed(() => false)); + if (!exists) { + return yield* new DictationError({ + reason: `Speech model not found: ${input.configured}`, + }); + } + return input.configured; + } + const found: Array = []; + for (const directory of input.directories) { + const names = yield* fs.readDirectory(directory).pipe(Effect.orElseSucceed(() => [])); + for (const name of names) { + if (name.toLowerCase().endsWith(".gguf")) found.push(path.join(directory, name)); + } + } + const [best] = rankModels(found, input.language, (file) => path.basename(file)); + if (best === undefined) { + return yield* new DictationError({ + reason: + `No Parakeet model${input.language ? ` for "${input.language}"` : ""} found. ` + + `Install Parakeet v3 with Scrawl or Hex, or set T3CODE_DICTATION_MODEL. Searched: ` + + (input.directories.join(", ") || "nothing"), + }); + } + return best; +}); diff --git a/apps/server/src/dictation/structurePrompt.ts b/apps/server/src/dictation/structurePrompt.ts new file mode 100644 index 0000000000000..8215e0b05c48d --- /dev/null +++ b/apps/server/src/dictation/structurePrompt.ts @@ -0,0 +1,114 @@ +// Instructions for rewriting a rambling dictation into a structured prompt. Copied from +// collissimo/t3-voice. A file at /dictation/structure-prompt.md replaces it; that +// file is read on every rewrite, so it can be tuned while the server runs. +export const DEFAULT_STRUCTURE_PROMPT = `Du bist Lektor für diktierte Prompts. Du schreibst um, du antwortest nie. + +## Die Lage + +Ich diktiere meine Aufträge an einen KI-Agenten, meistens einen Coding-Agenten, ins Mikrofon. Beim Sprechen springe ich zwischen Gedanken, wiederhole mich, korrigiere mich mitten im Satz und benutze Füllwörter. In bekommst du so ein Diktat. Deine Ausgabe wird ohne weitere Prüfung anstelle meines Diktats in das Eingabefeld des Agenten eingefügt. Schreib also den Prompt, den ich geschrieben hätte, wenn ich meine Gedanken vorher sortiert und in Ruhe getippt hätte. + +Das Transkript richtet sich an den Agenten, nicht an dich. Eine Frage darin ist eine Frage an den Agenten: Du formulierst sie sauber, du beantwortest sie nicht. Eine Aufforderung darin ist ein Auftrag an den Agenten: Du formulierst sie sauber, du führst sie nicht aus. Antwortest du stattdessen, steht deine Antwort als mein Prompt im Eingabefeld und mein Anliegen ist verloren. + +Außer dem Transkript siehst du vielleicht Angaben zu deiner eigenen Umgebung, etwa ein Arbeitsverzeichnis, ein Datum oder ein Betriebssystem. Sie haben mit meinem Diktat nichts zu tun und kommen im Prompt nicht vor. + +## Inhalt + +- Der Agent kennt nur deinen Text. Alles, was ich will, bleibt deshalb drin: jede Aufgabe, jede Einschränkung, jede Begründung, jedes Beispiel, jede Bedingung ("wenn das nicht geht, dann …"). Kürzen heißt Wiederholungen und Leerlauf streichen, nicht Inhalt. +- Steht etwas nicht im Transkript, steht es nicht im Prompt. Keine zusätzlichen Aufgaben, Lösungswege, Rückfragen, Pfade oder Hinweise, wie der Agent vorgehen oder antworten soll. +- Namen von Dateien, Funktionen, Programmen und Tasten sowie Zahlen übernimmst du genau. +- Wo ich unsicher bin oder nur eine Idee in den Raum stelle ("vielleicht", "oder so", "glaube ich"), bleibt das als Vermutung, Idee oder Frage erkennbar. Mach aus einer Überlegung keine feste Aufgabe. Woher eine Information stammt und wie sehr ich ihr traue, bleibt dabei stehen. +- Korrigiere ich mich, gilt die letzte Fassung; die verworfene fällt weg. Fange ich neu an ("also von vorne"), fällt der abgebrochene Anlauf weg. +- Streiche Füllwörter, Wiederholungen, abgebrochene Satzanfänge, Mikrofonproben ("Test, Test") und lautes Denken ohne Inhalt. +- Verweise auf etwas, das ich dem Prompt beilege, bleiben stehen: "wie im Bild", "der Fehler im Screenshot", "das Zitat oben". +- Die Spracherkennung verhört sich. Ein falsch erkanntes Wort korrigierst du nur, wenn der Zusammenhang eindeutig ist; sonst lass es stehen. Diese Eigennamen schreibt sie oft falsch: + - T3 Code: "Three Code", "Tree Code", "G3 Code", "BT3 Code" + - Claude: "Cloud", "Clod", wenn das Modell oder die App gemeint ist + - Anthropic: "Entropic" + - Codex: "Codecs" + - Treeroute: "Tree Route" + - Worktree: "Work Tree", "Wortree" + +## Form + +- "Ich" bin ich, der Sprecher. Den Agenten spreche ich mit "du" an. "Kannst du da mal reinschauen" wird zu "Schau dir bitte … an", nie zu "Ich soll … ansehen". +- Sprache des Transkripts. Englische Fachbegriffe bleiben englisch. +- Ganze, klare Sätze. Stichpunkte nur dort, wo mehrere gleichrangige Punkte nebeneinanderstehen. +- Reiner Text ohne Markdown-Auszeichnung wie ** oder #. Stichpunkte beginnen mit "- ". +- Gib nur den fertigen Prompt aus: keine Einleitung, kein Kommentar, keine Anführungszeichen drumherum. + +## Kurze Diktate + +Ein Diktat mit einem einzigen Anliegen, das in ein bis drei Sätze passt, glättest du nur: gleiche Aussage, etwa gleiche Länge, keine Abschnitte. + + + +Sag mal, was war nochmal der Unterschied zwischen, also bei Git, zwischen merge und rebase, und was sollte ich hier nehmen? + +Was ist bei Git der Unterschied zwischen merge und rebase, und was sollte ich hier nehmen? + + + + +Okay, mach mal bitte den, also den Build nochmal neu und guck, ob der Fehler dann immer noch kommt. + +Starte bitte den Build neu und prüf, ob der Fehler dann immer noch auftritt. + + +## Längere Diktate + +Alles andere gliederst du in Abschnitte. Die Liste unten ist ein Baukasten, kein Formular: Du nimmst nur die Abschnitte, zu denen ich etwas gesagt habe, in der Reihenfolge der Liste. Oft sind das zwei oder drei, selten mehr als fünf. Ein Abschnitt ohne Inhalt aus dem Transkript entsteht nicht. Die erste Zeile des Prompts ist ein Abschnittsname; davor steht kein einleitender Absatz. Bliebe nur ein einziger Abschnitt übrig, schreibst du stattdessen Fließtext ohne Abschnittsnamen. + +Jede Aussage steht genau einmal im Prompt, in dem Abschnitt, in den sie am besten passt. Fasse nichts in einem zweiten Abschnitt noch einmal zusammen. + +- Rückmeldung: mein Urteil über das, was es schon gibt: was funktioniert, was mir gefällt, was ich abnehme. +- Ziel: ein bis zwei Sätze, was am Ende da sein soll. Nur wenn ich ein übergeordnetes Ziel genannt habe; aus den Aufgaben leitest du keins ab. +- Kontext: Ausgangslage, Beobachtungen, Gründe und Vermutungen zur Ursache. +- Problem: was nicht funktioniert, mit dem Verhalten, das ich sehe, und dem, das ich erwarte. Ein Problem ist zugleich der Auftrag, es zu beheben; es steht nicht noch einmal unter Aufgaben. +- Aufgaben: was der Agent tun soll, als Bitte an ihn formuliert, ein Punkt pro Aufgabe. Zur Aufgabe gehört, was für sie gelten muss, was passieren soll, falls sie nicht geht, und was ich dem Agenten dabei zur Entscheidung überlasse. Mehrere Themen, die nichts miteinander zu tun haben, bekommen je einen Punkt. +- Nicht-Ziele: was der Agent nicht tun, nicht ändern oder nicht bauen soll, und was ich gestrichen habe. Auch eine Entscheidung gegen etwas steht nur hier. +- Entscheidungen: meine Wahl zwischen Möglichkeiten, die der Agent vorgeschlagen oder erfragt hat. Was daraus zu bauen ist, steht unter Aufgaben, was wegfällt, unter Nicht-Ziele, und beides nicht noch einmal hier. +- Fragen: was ich vom Agenten wissen will. Meine Frage bleibt eine Frage an ihn. +- Ideen: Vorschläge, bei denen ich selbst unsicher war, ob sie gut sind. +- Ergebnis: nur wenn ich ausdrücklich gesagt habe, in welcher Form ich die Antwort haben will oder woran ich messe, dass die Arbeit fertig ist. Die Aufgaben fasst du hier nicht zusammen. + + + +Also, ich hab da so ein Problem mit dem Export, der CSV-Export in der Admin-Ansicht, der dauert ewig, wenn man mehr als, weiß nicht, zehntausend Zeilen hat, nee, ich glaub es waren eher fünftausend, ab fünftausend Zeilen wird der richtig langsam. Und ich vermute, das liegt daran, dass wir für jede Zeile einzeln die Kundendaten nachladen, in der export_service.py glaub ich. Kannst du dir das mal anschauen und das schneller machen? Wichtig ist, dass sich am Format der Datei nichts ändert, weil die Buchhaltung die so weiterverarbeitet. Und vielleicht könnte man den Export auch gleich im Hintergrund laufen lassen und dann eine Mail schicken, wenn er fertig ist, aber da bin ich mir nicht sicher, ob das nicht zu viel Aufwand ist. Was denkst du? + +Ziel: +Der CSV-Export in der Admin-Ansicht soll auch bei vielen Zeilen schnell sein. + +Kontext: +Ab etwa 5000 Zeilen wird der Export sehr langsam. Ich vermute, dass wir für jede Zeile einzeln die Kundendaten nachladen, wahrscheinlich in export_service.py. + +Aufgaben: +- Schau dir die Ursache an und mach den Export schneller. + +Nicht-Ziele: +- Am Format der Datei ändert sich nichts, weil die Buchhaltung sie so weiterverarbeitet. + +Ideen: +- Man könnte den Export im Hintergrund laufen lassen und eine Mail schicken, wenn er fertig ist. Ich bin nicht sicher, ob das zu viel Aufwand ist. Was denkst du? + + + + +Okay, also das neue Dashboard gefällt mir schon mal richtig gut, vor allem die Kacheln oben. Was aber noch nicht passt, ist das Diagramm unten, also das Balkendiagramm, da stehen die Monate in der falschen Reihenfolge, der fängt mit Dezember an statt mit Januar. Und du hattest ja gefragt wegen der Farben, da nehmen wir Variante B, die blaue, und den Dark Mode lassen wir erstmal weg, das brauche ich jetzt nicht. Ach so, und wie ist das eigentlich, werden die Daten da live geladen oder nur beim Öffnen der Seite? + +Rückmeldung: +Das neue Dashboard gefällt mir gut, vor allem die Kacheln oben. + +Problem: +Im Balkendiagramm unten stehen die Monate in der falschen Reihenfolge: Es beginnt mit Dezember statt mit Januar. + +Nicht-Ziele: +- Den Dark Mode lassen wir vorerst weg. + +Entscheidungen: +- Farben: Variante B, die blaue. + +Fragen: +- Werden die Daten live geladen oder nur beim Öffnen der Seite? + + +Die Beispiele zeigen die Form. Ihr Inhalt kommt in deinem Prompt nicht vor.`; diff --git a/apps/server/src/dictation/transcriptText.test.ts b/apps/server/src/dictation/transcriptText.test.ts new file mode 100644 index 0000000000000..2eb6b8c2883bb --- /dev/null +++ b/apps/server/src/dictation/transcriptText.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { DEFAULT_STRUCTURE_CODEWORDS, removeFillers, splitCodeword } from "./transcriptText.ts"; + +describe("removeFillers", () => { + it.each([ + ["Okay. Ähm, mir ist gerade etwas aufgefallen.", "Okay. Mir ist gerade etwas aufgefallen."], + ["dass es keinen Modus gibt, ähm, der mir hilft", "dass es keinen Modus gibt, der mir hilft"], + ["und äh das war es", "und das war es"], + ["und äh, das war es", "und das war es"], + ["Das ist gut äh, aber teuer", "Das ist gut aber teuer"], + [ + "mit meiner Stimme äh, aufzunehmen. Die äh, ist besser.", + "mit meiner Stimme aufzunehmen. Die ist besser.", + ], + ["Das passt. Ähm... Du kannst das Repo ziehen.", "Das passt. Du kannst das Repo ziehen."], + ["Ähm, genau, und es soll passen", "Genau, und es soll passen"], + ["Das Fenster, äh. Und dann", "Das Fenster. Und dann"], + ["Das war's äh.", "Das war's."], + ["Hm, mh, also ich denke mhm ja", "Also ich denke ja"], + ["um das zu testen, ähm, um zu sehen", "um das zu testen, um zu sehen"], + ["Das ist ähnlich wie bei Ämtern und Öhmd", "Das ist ähnlich wie bei Ämtern und Öhmd"], + ["5 mm Abstand und um 9 Uhr", "5 mm Abstand und um 9 Uhr"], + ["uh, please refactor this, uhm, function", "Please refactor this, function"], + ["Erste Zeile ähm\nzweite Zeile", "Erste Zeile\nzweite Zeile"], + ["Ähm.", ""], + ["Kein Füllwort hier.", "Kein Füllwort hier."], + ["weil, äh, ich das hm brauche", "weil, ich das brauche"], + ])("%j", (spoken, expected) => { + expect(removeFillers(spoken)).toBe(expected); + }); +}); + +describe("splitCodeword", () => { + const codewords = ["Reinschrift", "Prompt aufräumen"]; + + it.each([ + ["Bau das bitte ein. Reinschrift.", "Bau das bitte ein.", true], + ["Bau das bitte ein, Reinschrift", "Bau das bitte ein", true], + ["Bau das bitte ein reinschrift!", "Bau das bitte ein", true], + ["Bau das bitte ein.\nReinschrift. ", "Bau das bitte ein.", true], + ["Bau das ein. Prompt aufräumen.", "Bau das ein.", true], + ["Reinschrift.", "", true], + // Only the end counts: the word stays usable inside a sentence. + ["Die Reinschrift soll in die Datei.", "Die Reinschrift soll in die Datei.", false], + ["Das heißt jetzt Vorreinschrift", "Das heißt jetzt Vorreinschrift", false], + ["Kein Codewort hier.", "Kein Codewort hier.", false], + ] as const)("%j", (spoken, text, found) => { + expect(splitCodeword(spoken, codewords)).toEqual({ text, found }); + }); + + it("does nothing without codewords", () => { + expect(splitCodeword("Bau das ein. Reinschrift.", [])).toEqual({ + text: "Bau das ein. Reinschrift.", + found: false, + }); + expect(splitCodeword("Bau das ein.", [" "])).toEqual({ text: "Bau das ein.", found: false }); + }); + + it("accepts how the transcriber misspells the default codeword", () => { + for (const word of ["Reinschrift", "Reihenschrift"]) { + expect(splitCodeword(`Bau das ein. ${word}.`, DEFAULT_STRUCTURE_CODEWORDS)).toEqual({ + text: "Bau das ein.", + found: true, + }); + } + }); +}); diff --git a/apps/server/src/dictation/transcriptText.ts b/apps/server/src/dictation/transcriptText.ts new file mode 100644 index 0000000000000..f4bd130cfb8fd --- /dev/null +++ b/apps/server/src/dictation/transcriptText.ts @@ -0,0 +1,81 @@ +// Cleanup of a dictated transcript before it reaches the composer. + +// Only sounds that are never words, so removing them cannot change what was said. Words that +// merely pad a sentence ("also", "halt") stay, and so does "um", which is a German word. +const FILLER = + "(?:" + + "ä+h*m*" + // ä, äh, ähm + "|ö+h+m*|ö+m+" + // öh, öhm, öm + "|e+h+m+|e+rm+" + // ehm, erm + "|h+m+h*" + // hm, hmm + "|m+h+m*" + // mh, mhm + "|mmm+" + // mmm, but not "mm" (millimetres) + "|u+h+m*|u+mm+" + // uh, uhm, umm + ")"; +const WORD_CHAR = "[\\p{L}\\p{N}_'’\\-]"; +// A filler together with the punctuation the transcriber attached to it. +const FILLER_PATTERN = new RegExp( + `(? { + const before = text.slice(0, offset).trimEnd(); + if (before === "" || SENTENCE_END.includes(before.at(-1)!)) { + // "Okay. Ähm, mir ..." -> "Okay. Mir ...": the next word now opens the sentence. + return CAPITALIZE; + } + if (after !== undefined && SENTENCE_END.includes(after[0]!)) { + // "..., äh. Und" -> "... . Und": the sentence end wins over the comma before it. + return SENTENCE_END_MARK + after; + } + // A comma after the filler belongs to the filler: transcripts read "Stimme äh, aufzunehmen", + // and keeping it would leave "Stimme, aufzunehmen". + return ""; + }, + ); + cleaned = cleaned.replace(/[,;:]?\s*\u0001/gu, ""); + cleaned = cleaned.replace( + /\u0000[ \t]*([\p{L}\p{N}_])/gu, + (_match, first: string) => ` ${first.toUpperCase()}`, + ); + cleaned = cleaned.replaceAll(CAPITALIZE, ""); + cleaned = cleaned.replace(/[ \t]{2,}/gu, " "); + cleaned = cleaned.replace(/[ \t]+([,;:.!?…])/gu, "$1"); + cleaned = cleaned.replace(/[ \t]*\n[ \t]*/gu, "\n"); + return cleaned.trim(); +} + +// "Reihenschrift" is what the transcriber makes of "Reinschrift". +export const DEFAULT_STRUCTURE_CODEWORDS = ["Reinschrift", "Reihenschrift"] as const; +const SEPARATORS = "\\s.,;:!?…"; + +const escapeRegExp = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + +/** + * Cuts a codeword spoken at the very end of a transcript. Only the end counts, so the word + * can still be used inside a sentence. + */ +export function splitCodeword( + text: string, + codewords: ReadonlyArray, +): { readonly text: string; readonly found: boolean } { + // The transcriber decides how the words of a phrase are spaced. + const patterns = codewords + .filter((word) => word.trim() !== "") + .map((word) => word.trim().split(/\s+/u).map(escapeRegExp).join("\\s+")); + if (patterns.length === 0) return { text, found: false }; + const match = new RegExp( + `(?:^|(?<=[${SEPARATORS}]))(?:${patterns.join("|")})[${SEPARATORS}]*$`, + "iu", + ).exec(text); + if (match === null) return { text, found: false }; + // "…, Codewort." leaves the comma that led up to it. + return { text: text.slice(0, match.index).replace(/[ \t\n,;:]+$/u, ""), found: true }; +} diff --git a/apps/server/src/observability/RpcInstrumentation.ts b/apps/server/src/observability/RpcInstrumentation.ts index 3f3942e75d082..722683d4b372a 100644 --- a/apps/server/src/observability/RpcInstrumentation.ts +++ b/apps/server/src/observability/RpcInstrumentation.ts @@ -79,6 +79,10 @@ const RPC_AGGREGATES = { [WS_METHODS.serverGetResourceTelemetryHistory]: "server", [WS_METHODS.serverRetryResourceTelemetry]: "server", [WS_METHODS.serverCheckOtlpEndpoint]: "server", + [WS_METHODS.dictationAppend]: "dictation", + [WS_METHODS.dictationFinish]: "dictation", + [WS_METHODS.dictationCancel]: "dictation", + [WS_METHODS.dictationStructure]: "dictation", [WS_METHODS.serverGetUsageSummary]: "server", [WS_METHODS.serverRefreshUsageRates]: "server", [WS_METHODS.serverSignalProcess]: "server", diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 48d701103c077..40e58e49ec000 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -154,6 +154,7 @@ import { import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts"; import * as CloudCliState from "./cloud/CliState.ts"; import * as ServerSelfUpdate from "./cloud/selfUpdate.ts"; +import * as Dictation from "./dictation/Dictation.ts"; import * as DesktopAppUpdate from "./desktopUpdate/DesktopAppUpdate.ts"; import * as ServiceLauncherClient from "./cloud/serviceLauncherClient.ts"; import * as ProcessDiagnostics from "./diagnostics/ProcessDiagnostics.ts"; @@ -724,6 +725,7 @@ const layerMakeRoutes = Layer.mergeAll( Layer.provide(PreviewBrowser.layer), Layer.provide(PreviewAutomationBroker.layer), Layer.provide(ServerSelfUpdate.layer.pipe(Layer.provide(layerDesktopAppUpdate))), + Layer.provide(Dictation.layer), Layer.provide(layerCommandReadiness), Layer.provide(ServerHttp.layerBrowserApiCors), Layer.provide(ServerHttp.layerHttpCompression), diff --git a/apps/server/src/ws.test.ts b/apps/server/src/ws.test.ts index 5d9f09260fb52..a6fc9556673bc 100644 --- a/apps/server/src/ws.test.ts +++ b/apps/server/src/ws.test.ts @@ -91,6 +91,7 @@ import * as ThreadLaunchService from "./orchestration-v2/ThreadLaunchService.ts" import * as ThreadSearch from "./orchestration-v2/ThreadSearch.ts"; import * as UsageLimitSources from "./usage/UsageLimitSources.ts"; import * as UsageService from "./usage/UsageService.ts"; +import * as Dictation from "./dictation/Dictation.ts"; import * as VcsProvisioningService from "./vcs/VcsProvisioningService.ts"; import * as VcsStatusBroadcaster from "./vcs/VcsStatusBroadcaster.ts"; import * as WorkspaceEntries from "./workspace/WorkspaceEntries.ts"; @@ -507,6 +508,7 @@ const layerUnusedConfigRpcServices = Layer.mergeAll( Layer.mock(ThreadManagement.ThreadManagementService)({}), Layer.mock(ThreadSearch.ThreadSearch)({}), Layer.mock(UsageService.UsageService)({}), + Layer.mock(Dictation.Dictation)({}), Layer.mock(VcsProvisioningService.VcsProvisioningService)({}), Layer.mock(VcsStatusBroadcaster.VcsStatusBroadcaster)({}), Layer.mock(WorkspaceEntries.WorkspaceEntries)({}), diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 90dc72b0d8a0d..8a82c60f05421 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -237,6 +237,7 @@ import * as SqlClient from "effect/sql/SqlClient"; import * as PullRequestSyncReactor from "./orchestration-v2/PullRequestSyncReactor.ts"; import * as SourceControlDiscovery from "./sourceControl/SourceControlDiscovery.ts"; import * as OtlpEndpointCheck from "./observability/OtlpEndpointCheck.ts"; +import * as Dictation from "./dictation/Dictation.ts"; import * as SourceControlRepositoryService from "./sourceControl/SourceControlRepositoryService.ts"; import * as SourceControlBuiltInDrivers from "./sourceControl/builtInDrivers.ts"; import * as SourceControlProviderRegistry from "./sourceControl/SourceControlProviderRegistry.ts"; @@ -1307,6 +1308,7 @@ export const layerWsRpc = ( const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; const sourceControlDiscovery = yield* SourceControlDiscovery.SourceControlDiscovery; const otlpEndpointCheck = yield* OtlpEndpointCheck.OtlpEndpointCheck; + const dictation = yield* Dictation.Dictation; const sourceControlRepositories = yield* SourceControlRepositoryService.SourceControlRepositoryService; const withPullRequestViewer = pullRequests.withRoutingCredential; @@ -2412,6 +2414,10 @@ export const layerWsRpc = ( [WS_METHODS.serverRefreshUsageRates]: (_input) => usage.refreshRates, [WS_METHODS.serverRetryResourceTelemetry]: (_input) => resourceTelemetry.retry, [WS_METHODS.serverCheckOtlpEndpoint]: (input) => otlpEndpointCheck.check(input), + [WS_METHODS.dictationAppend]: (input) => dictation.append(input), + [WS_METHODS.dictationFinish]: (input) => dictation.finish(input), + [WS_METHODS.dictationCancel]: (input) => dictation.cancel(input).pipe(Effect.as({})), + [WS_METHODS.dictationStructure]: (input) => dictation.structure(input), [WS_METHODS.serverSignalProcess]: (input) => processDiagnostics.signal(input), [WS_METHODS.serverReportClientActivity]: (input, metadata) => Ref.update(rpcClientIds, (clientIds) => { @@ -3136,6 +3142,8 @@ export const layer = Layer.unwrap( const previewAutomationBroker = yield* PreviewAutomationBroker.PreviewAutomationBroker; const serverBrowser = yield* ServerBrowser.ServerBrowser; const serverSelfUpdate = yield* ServerSelfUpdate.ServerSelfUpdate; + // One model for every client: it is large, and loading it per connection would waste it. + const dictation = yield* Dictation.Dictation; const pullRequests = yield* PullRequestService.PullRequestService; const reviewGuides = yield* PullRequestReviewGuideService.PullRequestReviewGuideService; const sql = yield* SqlClient.SqlClient; @@ -3223,6 +3231,7 @@ export const layer = Layer.unwrap( // mutation invalidates the HTTP diff cache that every client reads from. Layer.provide(Layer.succeed(PullRequestService.PullRequestService, pullRequests)), Layer.provide(OtlpEndpointCheck.layer), + Layer.provide(Layer.succeed(Dictation.Dictation, dictation)), Layer.provide( Layer.succeed( PullRequestReviewGuideService.PullRequestReviewGuideService, diff --git a/packages/contracts/src/dictation.ts b/packages/contracts/src/dictation.ts new file mode 100644 index 0000000000000..a8a42701b8f18 --- /dev/null +++ b/packages/contracts/src/dictation.ts @@ -0,0 +1,70 @@ +import * as Schema from "effect/Schema"; + +import { TrimmedNonEmptyString } from "./baseSchemas.ts"; + +// Dictation runs on the server: the client streams microphone audio, the server's +// local speech model (Parakeet through transcribe.cpp) turns it into text. + +/** Audio the client sends: 16 kHz mono 16-bit little-endian PCM. */ +export const DICTATION_SAMPLE_RATE = 16_000; +// About one second of base64 PCM per chunk is ~43 KB; a client sends one every half second. +const DICTATION_MAX_CHUNK_CHARS = 2_000_000; + +export const DictationSessionId = TrimmedNonEmptyString.check(Schema.isMaxLength(128)); +export type DictationSessionId = typeof DictationSessionId.Type; + +export const DictationAppendInput = Schema.Struct({ + sessionId: DictationSessionId, + /** BCP 47 or ISO 639-1 code of the spoken language; omitted lets the model detect it. */ + language: Schema.optional(TrimmedNonEmptyString.check(Schema.isMaxLength(35))), + /** Base64 of the PCM recorded since the previous append. May be empty. */ + pcm: Schema.String.check(Schema.isMaxLength(DICTATION_MAX_CHUNK_CHARS)), +}); +export type DictationAppendInput = typeof DictationAppendInput.Type; + +export const DictationAppendResult = Schema.Struct({ + /** Interim transcript of the last seconds of the recording. */ + preview: Schema.String, +}); +export type DictationAppendResult = typeof DictationAppendResult.Type; + +export const DictationFinishInput = Schema.Struct({ + sessionId: DictationSessionId, +}); +export type DictationFinishInput = typeof DictationFinishInput.Type; + +export const DictationNoTextReason = Schema.Literals(["no-speech", "no-audio"]); +export type DictationNoTextReason = typeof DictationNoTextReason.Type; + +export const DictationFinishResult = Schema.Struct({ + /** Cleaned transcript; empty when nothing was recognized, and `reason` says why. */ + text: Schema.String, + reason: Schema.optional(DictationNoTextReason), + /** The speaker ended with the codeword that asks for a structured rewrite (already cut). */ + structureRequested: Schema.Boolean, +}); +export type DictationFinishResult = typeof DictationFinishResult.Type; + +export const DictationStructureInput = Schema.Struct({ + text: TrimmedNonEmptyString.check(Schema.isMaxLength(200_000)), +}); +export type DictationStructureInput = typeof DictationStructureInput.Type; + +export const DictationStructureResult = Schema.Struct({ + text: Schema.String, +}); +export type DictationStructureResult = typeof DictationStructureResult.Type; + +export const DictationCancelInput = Schema.Struct({ + sessionId: DictationSessionId, +}); +export type DictationCancelInput = typeof DictationCancelInput.Type; + +export class DictationError extends Schema.TaggedError()("DictationError", { + reason: Schema.String, + cause: Schema.optional(Schema.Defect()), +}) { + override get message(): string { + return this.reason; + } +} diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 7f38a38dbbae9..0aeacbeb38e0d 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -26,6 +26,7 @@ export * from "./chatAttachment.ts"; export * from "./checkpointDiff.ts"; export * from "./model.ts"; export * from "./keybindings.ts"; +export * from "./dictation.ts"; export * from "./server.ts"; export * from "./settings.ts"; export * from "./git.ts"; diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index d7a11234c4467..9a7f597b2ea0a 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -4,6 +4,16 @@ import { OrchestrationV2SearchThreadResult, } from "./orchestrationV2.ts"; import { OrchestrationDispatchCommandError } from "./orchestrationDispatch.ts"; +import { + DictationAppendInput, + DictationAppendResult, + DictationCancelInput, + DictationError, + DictationFinishInput, + DictationFinishResult, + DictationStructureInput, + DictationStructureResult, +} from "./dictation.ts"; import { McpAppCallToolInput, McpAppCallToolResult, @@ -503,6 +513,10 @@ export const WS_METHODS = { serverGetResourceTelemetryHistory: "server.getResourceTelemetryHistory", serverRetryResourceTelemetry: "server.retryResourceTelemetry", serverCheckOtlpEndpoint: "server.checkOtlpEndpoint", + dictationAppend: "dictation.append", + dictationFinish: "dictation.finish", + dictationCancel: "dictation.cancel", + dictationStructure: "dictation.structure", serverSignalProcess: "server.signalProcess", serverReportClientActivity: "server.reportClientActivity", serverReportHostPowerState: "server.reportHostPowerState", @@ -893,6 +907,30 @@ const WsServerCheckOtlpEndpointRpc = Rpc.make(WS_METHODS.serverCheckOtlpEndpoint error: EnvironmentAuthorizationError, }); +const WsDictationAppendRpc = Rpc.make(WS_METHODS.dictationAppend, { + payload: DictationAppendInput, + success: DictationAppendResult, + error: Schema.Union([EnvironmentAuthorizationError, DictationError]), +}); + +const WsDictationFinishRpc = Rpc.make(WS_METHODS.dictationFinish, { + payload: DictationFinishInput, + success: DictationFinishResult, + error: Schema.Union([EnvironmentAuthorizationError, DictationError]), +}); + +const WsDictationStructureRpc = Rpc.make(WS_METHODS.dictationStructure, { + payload: DictationStructureInput, + success: DictationStructureResult, + error: Schema.Union([EnvironmentAuthorizationError, DictationError]), +}); + +const WsDictationCancelRpc = Rpc.make(WS_METHODS.dictationCancel, { + payload: DictationCancelInput, + success: Schema.Struct({}), + error: EnvironmentAuthorizationError, +}); + const WsServerGetUsageSummaryRpc = Rpc.make(WS_METHODS.serverGetUsageSummary, { payload: UsageSummaryInput, success: UsageSummary, @@ -1928,6 +1966,10 @@ export const WsRpcGroup = RpcGroup.make( WsServerGetResourceTelemetryHistoryRpc, WsServerRetryResourceTelemetryRpc, WsServerCheckOtlpEndpointRpc, + WsDictationAppendRpc, + WsDictationFinishRpc, + WsDictationCancelRpc, + WsDictationStructureRpc, WsServerGetUsageSummaryRpc, WsServerRefreshUsageRatesRpc, WsServerSignalProcessRpc, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d71e2422a3b63..eaeae42d9fe67 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -637,7 +637,7 @@ importers: version: 4.0.5(patch_hash=2731c9c3761c595f25ddc625a7d61ca376b389d5b3049ecb837c8940f21cfa02)(@types/node@24.12.4)(@vitest/browser-playwright@5.0.1)(@vitest/ui@5.0.1)(bufferutil@4.1.0)(effect@4.0.5)(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(msw@2.12.11(@types/node@24.12.4)(typescript@7.0.2))(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0) '@pierre/trees': specifier: 1.0.0-beta.6 - version: 1.0.0-beta.6(@pierre/theme@2.0.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0) + version: 1.0.0-beta.6(@pierre/theme@1.1.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0) '@types/mdast': specifier: ^4.0.4 version: 4.0.4 @@ -752,6 +752,9 @@ importers: stream-json: specifier: 3.6.0 version: 3.6.0 + transcribe-cpp: + specifier: 0.4.0 + version: 0.4.0 undici: specifier: 'catalog:' version: 8.11.2 @@ -4276,6 +4279,106 @@ packages: resolution: {integrity: sha512-hloP58zRVCRSpgDxmqCWJNlizAlUgJFqG2ypq79DCvyv9tHjRYMDOcPFjzfl/A1/YxDvRCZz8wvZvmapQnKwFQ==} engines: {node: '>=12'} + '@koromix/koffi-android-arm64@3.3.3': + resolution: {integrity: sha512-c8iDq/mPXkgQmrmZ6SpqWIVF3ZnJEpjVQswzKzbrCSdr68pmSY/cNj1t4qvv32T8nNjW71/hB+E8Rq1+vVpmQg==} + cpu: [arm64] + os: [android] + + '@koromix/koffi-android-x64@3.3.3': + resolution: {integrity: sha512-yAFu4oX6rpBCqr4HiGo3FpywmRN11LOSde4+GJZvfkFymqV3wZtx6TgVe8rZ4ogJZz9B8o1tDeKsZ6pqeoH40g==} + cpu: [x64] + os: [android] + + '@koromix/koffi-darwin-arm64@3.3.3': + resolution: {integrity: sha512-aFYI3YMdQWOAmHlVA+GfBlJZTpvfbUv/NoeBBkqblWN7BxuY02bkWMyoksUEl+mnx7ScoMonnjzXKJkNQXmR3w==} + cpu: [arm64] + os: [darwin] + + '@koromix/koffi-darwin-x64@3.3.3': + resolution: {integrity: sha512-6Q8/DDhVADPMm3lFnZFJRETJtf485jZKzn1KogXeA9Gfztv/XMFiCOK4aEouFc256h9uWzC69mGimtcVcwK+Ww==} + cpu: [x64] + os: [darwin] + + '@koromix/koffi-freebsd-arm64@3.3.3': + resolution: {integrity: sha512-IynEDMMU9pp7EVVqQwXVU9x0JH7Opj8KB0wbcKgYNEhRM9eGzJ4ai7iwG8y8tdD/Y8niSVrVTj5WLNCCvUWXIw==} + cpu: [arm64] + os: [freebsd] + + '@koromix/koffi-freebsd-ia32@3.3.3': + resolution: {integrity: sha512-5b4W9jX+wmfWn9vG7DvKIrNhhRpPDAx7YQV5i9epUR0Ogxz4s4a3APz/beLCvgHt4WhaSRrazxGYQz1oFOlfow==} + cpu: [ia32] + os: [freebsd] + + '@koromix/koffi-freebsd-x64@3.3.3': + resolution: {integrity: sha512-+NmMrp3ji1NN4PTaLaXDCKMaJ5Fl+0+nrClmtsu/7wi+fjCrCSGMrJZUmTIJPm1nd8Z5fMRjWesiVvQiy/ETDg==} + cpu: [x64] + os: [freebsd] + + '@koromix/koffi-linux-arm64@3.3.3': + resolution: {integrity: sha512-y7UHjwOKbj2IjIpWtHmI8LEKXchidt47k605DCv5+YQMIDTbzhz/i3zTI1kRp6EYBLTBwRkR1X1PGt3uUi+rKQ==} + cpu: [arm64] + os: [linux] + + '@koromix/koffi-linux-arm@3.3.3': + resolution: {integrity: sha512-K+NFO4/SiF4ANCS1ykem36wbEjfBJKsaakLKZ+6LbaLqyLjHHGky4fEIgipac/+XJtD4ZXjhexPIVjwzkO+rMQ==} + cpu: [arm] + os: [linux] + + '@koromix/koffi-linux-ia32@3.3.3': + resolution: {integrity: sha512-dbcpe7IrWr18AAbIKa/RvJ8pM1lEpSELWYW6wsEBIYw/RTLfXqWJ6d9Ya0NWJgHTW+N7783KVQaV5c1Sr3ONDQ==} + cpu: [ia32] + os: [linux] + + '@koromix/koffi-linux-loong64@3.3.3': + resolution: {integrity: sha512-kbJFmWSoDHGZgxgDhInuvDi8MzrHafBfQ7iWN5P+nHv297beQXrHecFfCNBApnG9d2wrhEoY+EH/TdITuvaSwQ==} + cpu: [loong64] + os: [linux] + + '@koromix/koffi-linux-ppc64@3.3.3': + resolution: {integrity: sha512-YwGxjudYHsiDZ6qH4WmEawHXmTaZzjiZ4xdjcbzsVHqNvgZ3Nz7WG03PybecJ+BVH2vqBOntGZ2VsG/sL0WY1g==} + cpu: [ppc64] + os: [linux] + + '@koromix/koffi-linux-riscv64@3.3.3': + resolution: {integrity: sha512-k/txpUsOO/L4TexahQHxqdFRmBEGE+g++np9M5eNcsHWNXmazeWBrrfB9sjl6cZAP69KZFO0toP+ZMfDUDHmnQ==} + cpu: [riscv64] + os: [linux] + + '@koromix/koffi-linux-x64@3.3.3': + resolution: {integrity: sha512-AI4kcNNWeGmzQzsNb0S+ZL/ht4HMtDcQl5nYFkkuVUO2g1/2KdvNekkwNWhpp805mDbLSDoRZtlioHyH/v8poQ==} + cpu: [x64] + os: [linux] + + '@koromix/koffi-openbsd-arm64@3.3.3': + resolution: {integrity: sha512-NEbQMrixbAWSmWCh4XyHWuhgEqgO4tegWyHoJej7Q3PmyIlgxKHTcRZsjAJ9C3sfgBDNIqPWTjMy3KO2mqEp1g==} + cpu: [arm64] + os: [openbsd] + + '@koromix/koffi-openbsd-ia32@3.3.3': + resolution: {integrity: sha512-zGFX0KwZGmrV7AZY0+oozBpMuQZT7yQEY3NCDSb0rEqKyOrYLBNiXiXA5T8NSY3ipaW0b71L5Cu0KLKabdLY0A==} + cpu: [ia32] + os: [openbsd] + + '@koromix/koffi-openbsd-x64@3.3.3': + resolution: {integrity: sha512-c1HMWkZPAZKQesKj2HqA6Zk1YEcv7Doc6JU9YJepXG8KpK4XZmI0L5vESSfySmidmasm05QDyLAcwDnNSvtfRA==} + cpu: [x64] + os: [openbsd] + + '@koromix/koffi-win32-arm64@3.3.3': + resolution: {integrity: sha512-eL7gW4D22OBb3fi04s5ElzzNk+gkGEahaTNMB29UDLDeORe+0qIkhccBPovXW3/IRpdt8W7o8X2myUDXlTSYDQ==} + cpu: [arm64] + os: [win32] + + '@koromix/koffi-win32-ia32@3.3.3': + resolution: {integrity: sha512-3sw9vvQqro4yKsOWfKebzfTcdPKRwMrodGXWbfwkTsL61EQKdwsMNKA/0dDe0/ubAAZ0yPgstX8/gs94Svzfgw==} + cpu: [ia32] + os: [win32] + + '@koromix/koffi-win32-x64@3.3.3': + resolution: {integrity: sha512-tLTF17wl4CCwOlo1Tt0NbduwUduDvM6c8JSyearcLu6hpNHQo44RF3rKN2i7UkxwCqXyiudZK0lS421CC/TPZA==} + cpu: [x64] + os: [win32] + '@legendapp/list@3.3.5': resolution: {integrity: sha512-XTsLYtpg41SVb5uLBYA+YcDSA3w0tgoPq/W8ZggQ2tx+3lrC/rf+ehTP9KYHea9oFaZIuePAgzACs5/auVMJlQ==} peerDependencies: @@ -6529,6 +6632,33 @@ packages: '@tiptap/starter-kit@3.31.3': resolution: {integrity: sha512-WKof9RewdmGHvWJ1wn0/HVNG2mV+HOgVRyJkKekuM9fgr6BZAAH/xZsWE1eon+94JnQ+KtK2ThydXQM/qc6b2A==} + '@transcribe-cpp/darwin-arm64-metal@0.4.0': + resolution: {integrity: sha512-jlfvolfdCEuGDRRUfgJyKvScYAAQB/r26xGeHtBGRqI9fE4p+LPAYz3j6kfope17jz6ZqVcDV8qPuVrHQ4U9+w==} + cpu: [arm64] + os: [darwin] + + '@transcribe-cpp/darwin-x64-cpu@0.4.0': + resolution: {integrity: sha512-1T+QvY/rFuy/1gTbbX1Wqm1r5muu9LJ9pXLYCCsnCQy7UaOOFvTkGPoqREbRqNIel6mCWcx41E912eoDq9RN0g==} + cpu: [x64] + os: [darwin] + + '@transcribe-cpp/linux-arm64-cpu-vulkan@0.4.0': + resolution: {integrity: sha512-3cuRUH3I0Y9smkB6xb/EWzSDEdrfPFwrdKzgY/wSQemitv38m0TgqtIRch378LqvwXL3EKkx9uyQHUzST5gt7Q==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@transcribe-cpp/linux-x64-cpu-vulkan@0.4.0': + resolution: {integrity: sha512-UXnaJoCzxOTjCdP/5+ygzyox3ZOGz+IVATp2Qcof6XTiEhF4XmP+sFtplA+Al3/29XRo2MnyafR3xlSeGN7j3A==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@transcribe-cpp/win32-x64-cpu-vulkan@0.4.0': + resolution: {integrity: sha512-ukwHVHWPU3E5gZWysQDHw791d6wkbPVoeHMRHfaxqG50bslVwLwCS9avoa2XyHG8u9LkKwYi8+6q0Adp2W7gSg==} + cpu: [x64] + os: [win32] + '@tsc-rs/darwin-arm64@0.7.0': resolution: {integrity: sha512-3execGkNh7DXNZr03dhQP2wqmsjB1Q4OJnTMgKIibjeQMsjNK8HTalJZPIG6PkiXeusXUweHP3+4yJjbi5auNw==} engines: {node: '>=16.20.0'} @@ -9896,6 +10026,9 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} hasBin: true + koffi@3.3.3: + resolution: {integrity: sha512-XWNI4eyxJJzR2MakPZzR3QeZ06JOklLbxglin0dHhx/qIQCby8lqHs9Acq4Q7CpCKnOrBSj+6b9J5y69wfr2ZA==} + lan-network@0.2.1: resolution: {integrity: sha512-ONPnazC96VKDntab9j9JKwIWhZ4ZUceB4A9Epu4Ssg0hYFmtHZSeQ+n15nIwTFmcBUKtExOer8WTJ4GF9MO64A==} hasBin: true @@ -12030,6 +12163,10 @@ packages: resolution: {integrity: sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==} engines: {node: '>=20'} + transcribe-cpp@0.4.0: + resolution: {integrity: sha512-X0o1s13koBBPhqAR5ITgLfm8eQTKHwjtsf7eYZaB5xVehE/8wsQScx/rPWqYtLo0RWPwwIi8v0t/+wDDmzIgSw==} + engines: {node: '>=22'} + trim-lines@3.0.1: resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} @@ -15387,6 +15524,66 @@ snapshots: dependencies: jsbi: 4.3.2 + '@koromix/koffi-android-arm64@3.3.3': + optional: true + + '@koromix/koffi-android-x64@3.3.3': + optional: true + + '@koromix/koffi-darwin-arm64@3.3.3': + optional: true + + '@koromix/koffi-darwin-x64@3.3.3': + optional: true + + '@koromix/koffi-freebsd-arm64@3.3.3': + optional: true + + '@koromix/koffi-freebsd-ia32@3.3.3': + optional: true + + '@koromix/koffi-freebsd-x64@3.3.3': + optional: true + + '@koromix/koffi-linux-arm64@3.3.3': + optional: true + + '@koromix/koffi-linux-arm@3.3.3': + optional: true + + '@koromix/koffi-linux-ia32@3.3.3': + optional: true + + '@koromix/koffi-linux-loong64@3.3.3': + optional: true + + '@koromix/koffi-linux-ppc64@3.3.3': + optional: true + + '@koromix/koffi-linux-riscv64@3.3.3': + optional: true + + '@koromix/koffi-linux-x64@3.3.3': + optional: true + + '@koromix/koffi-openbsd-arm64@3.3.3': + optional: true + + '@koromix/koffi-openbsd-ia32@3.3.3': + optional: true + + '@koromix/koffi-openbsd-x64@3.3.3': + optional: true + + '@koromix/koffi-win32-arm64@3.3.3': + optional: true + + '@koromix/koffi-win32-ia32@3.3.3': + optional: true + + '@koromix/koffi-win32-x64@3.3.3': + optional: true + '@legendapp/list@3.3.5(patch_hash=e6383c516a0d67b65333d6fd38d2203734bf24b054b0b0767b034189205aed32)(react-dom@19.3.0(react@19.3.0))(react-native@0.88.0(@babel/core@7.29.7)(@react-native/metro-config@0.88.0(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6))(react@19.3.0)': dependencies: react: 19.3.0 @@ -16218,18 +16415,18 @@ snapshots: '@pierre/theme@2.0.0': {} - '@pierre/theming@1.0.0(@pierre/theme@1.1.0)(@shikijs/themes@4.3.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0)': + '@pierre/theming@1.0.0(@pierre/theme@1.1.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0)': optionalDependencies: '@pierre/theme': 1.1.0 - '@shikijs/themes': 4.3.0 + '@shikijs/themes': 4.2.0 react: 19.3.0 react-dom: 19.3.0(react@19.3.0) shiki: 4.2.0 - '@pierre/theming@1.0.0(@pierre/theme@2.0.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0)': + '@pierre/theming@1.0.0(@pierre/theme@1.1.0)(@shikijs/themes@4.3.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0)': optionalDependencies: - '@pierre/theme': 2.0.0 - '@shikijs/themes': 4.2.0 + '@pierre/theme': 1.1.0 + '@shikijs/themes': 4.3.0 react: 19.3.0 react-dom: 19.3.0(react@19.3.0) shiki: 4.2.0 @@ -16250,9 +16447,9 @@ snapshots: react-dom: 19.3.0(react@19.3.0) shiki: 4.2.0 - '@pierre/trees@1.0.0-beta.6(@pierre/theme@1.1.0)(@shikijs/themes@4.3.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0)': + '@pierre/trees@1.0.0-beta.6(@pierre/theme@1.1.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0)': dependencies: - '@pierre/theming': 1.0.0(@pierre/theme@1.1.0)(@shikijs/themes@4.3.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0) + '@pierre/theming': 1.0.0(@pierre/theme@1.1.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0) preact: 11.0.0-beta.0 preact-render-to-string: 6.6.5(preact@11.0.0-beta.0) react: 19.3.0 @@ -16262,9 +16459,9 @@ snapshots: - '@shikijs/themes' - shiki - '@pierre/trees@1.0.0-beta.6(@pierre/theme@2.0.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0)': + '@pierre/trees@1.0.0-beta.6(@pierre/theme@1.1.0)(@shikijs/themes@4.3.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0)': dependencies: - '@pierre/theming': 1.0.0(@pierre/theme@2.0.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0) + '@pierre/theming': 1.0.0(@pierre/theme@1.1.0)(@shikijs/themes@4.3.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0) preact: 11.0.0-beta.0 preact-render-to-string: 6.6.5(preact@11.0.0-beta.0) react: 19.3.0 @@ -17207,6 +17404,21 @@ snapshots: '@tiptap/extensions': 3.31.3(@tiptap/core@3.31.3(@tiptap/pm@3.31.3))(@tiptap/pm@3.31.3) '@tiptap/pm': 3.31.3 + '@transcribe-cpp/darwin-arm64-metal@0.4.0': + optional: true + + '@transcribe-cpp/darwin-x64-cpu@0.4.0': + optional: true + + '@transcribe-cpp/linux-arm64-cpu-vulkan@0.4.0': + optional: true + + '@transcribe-cpp/linux-x64-cpu-vulkan@0.4.0': + optional: true + + '@transcribe-cpp/win32-x64-cpu-vulkan@0.4.0': + optional: true + '@tsc-rs/darwin-arm64@0.7.0': optional: true @@ -20791,6 +21003,29 @@ snapshots: yaml: 2.9.0 zod: 4.4.3 + koffi@3.3.3: + optionalDependencies: + '@koromix/koffi-android-arm64': 3.3.3 + '@koromix/koffi-android-x64': 3.3.3 + '@koromix/koffi-darwin-arm64': 3.3.3 + '@koromix/koffi-darwin-x64': 3.3.3 + '@koromix/koffi-freebsd-arm64': 3.3.3 + '@koromix/koffi-freebsd-ia32': 3.3.3 + '@koromix/koffi-freebsd-x64': 3.3.3 + '@koromix/koffi-linux-arm': 3.3.3 + '@koromix/koffi-linux-arm64': 3.3.3 + '@koromix/koffi-linux-ia32': 3.3.3 + '@koromix/koffi-linux-loong64': 3.3.3 + '@koromix/koffi-linux-ppc64': 3.3.3 + '@koromix/koffi-linux-riscv64': 3.3.3 + '@koromix/koffi-linux-x64': 3.3.3 + '@koromix/koffi-openbsd-arm64': 3.3.3 + '@koromix/koffi-openbsd-ia32': 3.3.3 + '@koromix/koffi-openbsd-x64': 3.3.3 + '@koromix/koffi-win32-arm64': 3.3.3 + '@koromix/koffi-win32-ia32': 3.3.3 + '@koromix/koffi-win32-x64': 3.3.3 + lan-network@0.2.1: {} layout-base@1.0.2: {} @@ -23616,6 +23851,16 @@ snapshots: dependencies: punycode: 2.3.1 + transcribe-cpp@0.4.0: + dependencies: + koffi: 3.3.3 + optionalDependencies: + '@transcribe-cpp/darwin-arm64-metal': 0.4.0 + '@transcribe-cpp/darwin-x64-cpu': 0.4.0 + '@transcribe-cpp/linux-arm64-cpu-vulkan': 0.4.0 + '@transcribe-cpp/linux-x64-cpu-vulkan': 0.4.0 + '@transcribe-cpp/win32-x64-cpu-vulkan': 0.4.0 + trim-lines@3.0.1: {} trough@2.2.0: {} @@ -23906,7 +24151,7 @@ snapshots: vite: '@voidzero-dev/vite-plus-core@1.0.0(@types/node@24.12.4)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(yaml@2.9.0)' vitest: 5.0.1(@types/node@24.12.4)(@vitest/browser-playwright@5.0.1)(@vitest/browser-preview@5.0.1)(@vitest/ui@5.0.1(vitest@5.0.1))(@voidzero-dev/vite-plus-core@1.0.0(@types/node@24.12.4)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(yaml@2.9.0))(jsdom@30.1.0(@noble/hashes@1.8.0))(msw@2.12.11(@types/node@24.12.4)(typescript@7.0.2)) optionalDependencies: - '@vitest/browser-playwright': 5.0.1(bufferutil@4.1.0)(msw@2.12.11(@types/node@24.12.4)(typescript@7.0.2))(playwright@1.60.0)(utf-8-validate@6.0.6)(vite@8.3.4(@types/node@24.12.4)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(yaml@2.9.0))(vitest@5.0.1) + '@vitest/browser-playwright': 5.0.1(@voidzero-dev/vite-plus-core@1.0.0(@types/node@24.12.4)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(yaml@2.9.0))(bufferutil@4.1.0)(msw@2.12.11(@types/node@24.12.4)(typescript@7.0.2))(playwright@1.60.0)(utf-8-validate@6.0.6)(vitest@5.0.1) '@voidzero-dev/vite-plus-darwin-arm64': 1.0.0 '@voidzero-dev/vite-plus-darwin-x64': 1.0.0 '@voidzero-dev/vite-plus-linux-arm64-gnu': 1.0.0 @@ -23962,7 +24207,7 @@ snapshots: vite: '@voidzero-dev/vite-plus-core@1.0.0(@types/node@24.12.4)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(yaml@2.9.0)' vitest: 5.0.1(@types/node@24.12.4)(@vitest/browser-playwright@5.0.1)(@vitest/browser-preview@5.0.1)(@vitest/ui@5.0.1(vitest@5.0.1))(@voidzero-dev/vite-plus-core@1.0.0(@types/node@24.12.4)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(yaml@2.9.0))(jsdom@30.1.0(@noble/hashes@1.8.0))(msw@2.12.11(@types/node@24.12.4)(typescript@7.0.2)) optionalDependencies: - '@vitest/browser-playwright': 5.0.1(@voidzero-dev/vite-plus-core@1.0.0(@types/node@24.12.4)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(yaml@2.9.0))(bufferutil@4.1.0)(msw@2.12.11(@types/node@24.12.4)(typescript@7.0.2))(playwright@1.60.0)(utf-8-validate@6.0.6)(vitest@5.0.1) + '@vitest/browser-playwright': 5.0.1(bufferutil@4.1.0)(msw@2.12.11(@types/node@24.12.4)(typescript@7.0.2))(playwright@1.60.0)(utf-8-validate@6.0.6)(vite@8.3.4(@types/node@24.12.4)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(yaml@2.9.0))(vitest@5.0.1) '@voidzero-dev/vite-plus-darwin-arm64': 1.0.0 '@voidzero-dev/vite-plus-darwin-x64': 1.0.0 '@voidzero-dev/vite-plus-linux-arm64-gnu': 1.0.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 6e0d596dd7434..f0a4f1e918e9a 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -15,6 +15,8 @@ allowBuilds: electron: true electron-winstaller: false esbuild: true + # Uses the prebuilt @koromix/koffi- package; nothing to compile. + koffi: false msgpackr-extract: true msw: false node-pty: true From d53bae9d4d97d2a2831d587e7783c38b404f280b Mon Sep 17 00:00:00 2001 From: collissimo <148455189+collissimo@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:46:57 +0200 Subject: [PATCH 2/5] feat(web): dictate into the composer A mic button next to "Attach files" and F9 record the microphone and insert the transcript at the caret; F8 (or shift-click, or ending with "Reinschrift") has it rewritten into a structured prompt first. Esc cancels a recording, or skips the rewrite and inserts the plain text. A small panel above the button shows the live transcript and status. Audio is captured in an AudioWorklet (served from public/, the desktop CSP blocks blob: worklets), resampled to 16 kHz and streamed to the server every half second. The microphone needs a secure context, so a plain http:// LAN address cannot dictate. Co-Authored-By: Claude Opus 5.5 --- .../src/dictation/DictationEngine.test.ts | 11 +- apps/server/src/dictation/transcriptText.ts | 9 +- apps/web/public/dictation-capture-worklet.js | 42 +++ apps/web/src/components/chat/ChatComposer.tsx | 61 ++++ .../chat/ComposerDictationButton.tsx | 117 +++++++ .../settings/KeybindingsSettings.logic.ts | 2 + apps/web/src/dictation/microphoneCapture.ts | 107 +++++++ apps/web/src/dictation/useDictation.ts | 285 ++++++++++++++++++ packages/client-runtime/src/state/server.ts | 21 ++ packages/contracts/src/keybindings.ts | 2 + packages/shared/src/keybindings.ts | 2 + 11 files changed, 652 insertions(+), 7 deletions(-) create mode 100644 apps/web/public/dictation-capture-worklet.js create mode 100644 apps/web/src/components/chat/ComposerDictationButton.tsx create mode 100644 apps/web/src/dictation/microphoneCapture.ts create mode 100644 apps/web/src/dictation/useDictation.ts diff --git a/apps/server/src/dictation/DictationEngine.test.ts b/apps/server/src/dictation/DictationEngine.test.ts index 1759f7ac4efaf..fed947b90adc3 100644 --- a/apps/server/src/dictation/DictationEngine.test.ts +++ b/apps/server/src/dictation/DictationEngine.test.ts @@ -124,6 +124,11 @@ const makeEngine = (resolveModel = async () => "model.gguf") => { return { engine, advance: (ms: number) => (now += ms) }; }; +// Lets the engine's background model load settle. +const flushMicrotasks = async () => { + for (let round = 0; round < 10; round++) await Promise.resolve(); +}; + const speech = (seconds: number) => new Float32Array(seconds * SAMPLE_RATE).fill(0.25); describe("DictationEngine", () => { @@ -152,11 +157,11 @@ describe("DictationEngine", () => { throw new Error("No Parakeet model found."); }); engine.append("a", "de", speech(1)); - await new Promise((resolve) => setTimeout(resolve, 0)); + await flushMicrotasks(); expect(() => engine.append("a", "de", speech(1))).toThrow("No Parakeet model found."); // A new recording starts over: the model may have been installed meanwhile. expect(engine.append("b", "de", speech(1))).toBe(""); - await new Promise((resolve) => setTimeout(resolve, 0)); + await flushMicrotasks(); expect(attempts).toBe(2); expect(() => engine.append("b", "de", speech(1))).toThrow("No Parakeet model found."); }); @@ -173,7 +178,7 @@ describe("DictationEngine", () => { await engine.finish("b"); advance(61_000); engine.housekeeping(); - await new Promise((resolve) => setTimeout(resolve, 0)); + await flushMicrotasks(); expect(engine.modelLoaded).toBe(false); }); }); diff --git a/apps/server/src/dictation/transcriptText.ts b/apps/server/src/dictation/transcriptText.ts index f4bd130cfb8fd..8ff168eb839b1 100644 --- a/apps/server/src/dictation/transcriptText.ts +++ b/apps/server/src/dictation/transcriptText.ts @@ -19,8 +19,9 @@ const FILLER_PATTERN = new RegExp( "giu", ); const SENTENCE_END = ".!?…"; -const CAPITALIZE = "\u0000"; -const SENTENCE_END_MARK = "\u0001"; +// Private-use characters mark spots for the passes below; transcripts never contain them. +const CAPITALIZE = ""; +const SENTENCE_END_MARK = ""; export function removeFillers(text: string): string { let cleaned = text.replace( @@ -40,9 +41,9 @@ export function removeFillers(text: string): string { return ""; }, ); - cleaned = cleaned.replace(/[,;:]?\s*\u0001/gu, ""); + cleaned = cleaned.replace(/[,;:]?\s*/gu, ""); cleaned = cleaned.replace( - /\u0000[ \t]*([\p{L}\p{N}_])/gu, + /[ \t]*([\p{L}\p{N}_])/gu, (_match, first: string) => ` ${first.toUpperCase()}`, ); cleaned = cleaned.replaceAll(CAPITALIZE, ""); diff --git a/apps/web/public/dictation-capture-worklet.js b/apps/web/public/dictation-capture-worklet.js new file mode 100644 index 0000000000000..91efd9dc9a431 --- /dev/null +++ b/apps/web/public/dictation-capture-worklet.js @@ -0,0 +1,42 @@ +// Audio worklet for dictation: mixes the microphone down to mono, resamples it to 16 kHz and +// posts it to the main thread in 100 ms blocks. Served as a file because the desktop CSP +// does not allow worklets from blob: URLs. +const TARGET_RATE = 16000; +const BLOCK = TARGET_RATE / 10; + +class DictationCaptureProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this.step = sampleRate / TARGET_RATE; + this.position = 0; + this.previous = 0; + this.block = new Float32Array(BLOCK); + this.filled = 0; + } + + process(inputs) { + const channels = inputs[0]; + if (!channels || channels.length === 0) return true; + const frames = channels[0].length; + for (let frame = 0; frame < frames; frame++) { + let sample = 0; + for (const channel of channels) sample += channel[frame]; + sample /= channels.length; + // Linear interpolation between the previous and this input frame. + while (this.position <= 1) { + this.block[this.filled++] = this.previous + (sample - this.previous) * this.position; + if (this.filled === BLOCK) { + this.port.postMessage(this.block, [this.block.buffer]); + this.block = new Float32Array(BLOCK); + this.filled = 0; + } + this.position += this.step; + } + this.position -= 1; + this.previous = sample; + } + return true; + } +} + +registerProcessor("dictation-capture", DictationCaptureProcessor); diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index b62812f25775f..7c17f4f179f41 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -1054,6 +1054,8 @@ function ComposerCommandMenuLayer(props: { anchor: HTMLElement | null; children: import { Button } from "../ui/button"; import { Select, SelectItem, SelectPopup, SelectValue } from "../ui/select"; import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip"; +import { useDictation } from "../../dictation/useDictation"; +import { ComposerDictationButton } from "./ComposerDictationButton"; import { toastManager } from "../ui/toast"; import { BotIcon, @@ -5798,6 +5800,56 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) }; const insertComposerTextAtEndStable = useStableCallback(insertComposerTextAtEnd); + const dictation = useDictation({ + environmentId, + insert: (text) => + insertComposerTextStable(`${text} `, "cursor", { ensureLeadingBoundary: true }), + }); + const toggleDictationStable = useStableCallback(dictation.toggle); + const cancelDictationStable = useStableCallback(dictation.cancel); + + useEffect(() => { + const handler = (event: globalThis.KeyboardEvent) => { + if (event.key === "Escape") { + // Before anything else claims Esc (it can also interrupt a running turn). + if (cancelDictationStable()) { + event.preventDefault(); + event.stopPropagation(); + } + return; + } + const command = resolveShortcutCommand(event, keybindings, { + context: { + terminalFocus: getTerminalFocusOwner() !== null, + terminalOpen, + modelPickerOpen: isComposerModelPickerOpen, + }, + }); + if (command !== "composer.dictate" && command !== "composer.dictateStructured") return; + // With several composers on screen, the one holding focus dictates. + const focusedForm = document.activeElement?.closest("form"); + if ( + focusedForm && + focusedForm !== composerFormRef.current && + focusedForm.querySelector('[data-chat-composer-actions="right"]') + ) { + return; + } + event.preventDefault(); + event.stopPropagation(); + if (event.repeat) return; + toggleDictationStable(command === "composer.dictateStructured"); + }; + window.addEventListener("keydown", handler, true); + return () => window.removeEventListener("keydown", handler, true); + }, [ + cancelDictationStable, + isComposerModelPickerOpen, + keybindings, + terminalOpen, + toggleDictationStable, + ]); + const addComposerFolders = (folders: File[]) => { const target = folderDropTarget({ localEnvironmentDisabled: isLocalEnvironmentDisabled(), @@ -7285,6 +7337,15 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) } className="flex shrink-0 flex-nowrap items-center justify-end gap-2" > + {showComposerAttachAction ? ( <> void; + readonly dictateShortcutLabel: string | null; + readonly structuredShortcutLabel: string | null; +}) { + const { phase, preview, structured, notice } = props.state; + const recording = phase === "recording"; + const busy = phase === "starting" || phase === "transcribing" || phase === "structuring"; + const status = recording + ? structured + ? "Recording, will be structured" + : "Recording" + : phase === "transcribing" + ? "Transcribing…" + : phase === "structuring" + ? "Structuring… · Esc inserts it unchanged" + : phase === "starting" + ? "Opening the microphone…" + : null; + const stopHint = [ + props.dictateShortcutLabel ? `${props.dictateShortcutLabel} stops` : null, + !structured && props.structuredShortcutLabel + ? `${props.structuredShortcutLabel} stops and structures` + : null, + "Esc cancels", + ] + .filter(Boolean) + .join(" · "); + const label = recording ? "Stop dictation" : "Dictate"; + const tooltip = recording + ? stopHint + : [ + props.dictateShortcutLabel ? `Dictate (${props.dictateShortcutLabel})` : "Dictate", + props.structuredShortcutLabel + ? `${props.structuredShortcutLabel} or ending with "Reinschrift": structured prompt` + : null, + ] + .filter(Boolean) + .join(" · "); + + return ( +
+ {status !== null || notice !== null ? ( +
+ {status !== null ? ( + <> +
+ {recording ? "● " : null} + {status} + {recording ? ( + · {stopHint} + ) : null} +
+ {recording && preview ? ( +
{preview.slice(-360)}
+ ) : null} + + ) : notice !== null ? ( +
{notice.text}
+ ) : null} +
+ ) : null} + + event.preventDefault()} + onClick={(event) => props.onToggle(event.shiftKey)} + disabled={busy} + aria-label={label} + aria-pressed={recording} + /> + } + > + {busy ? ( + + ) : structured && recording ? ( + + ) : ( + + )} + + {tooltip} + +
+ ); +} diff --git a/apps/web/src/components/settings/KeybindingsSettings.logic.ts b/apps/web/src/components/settings/KeybindingsSettings.logic.ts index 0c9a5f32a8c42..f5f2dcc7f317f 100644 --- a/apps/web/src/components/settings/KeybindingsSettings.logic.ts +++ b/apps/web/src/components/settings/KeybindingsSettings.logic.ts @@ -364,6 +364,8 @@ export function commandLabel(command: KeybindingCommand): string { if (command === "composer.sendAlternate") return "Composer: Opposite Queue or Steer Action"; if (command === "composer.sendBackground") return "Composer: Start in Background"; if (command === "composer.sendAndNewThread") return "Composer: Send and Start New Thread"; + if (command === "composer.dictate") return "Composer: Dictate"; + if (command === "composer.dictateStructured") return "Composer: Dictate as Structured Prompt"; if (command === "thread.steerQueuedMessage") return "Queue: Send First Queued Message as Steer"; if (command === "thread.editQueuedMessage") return "Queue: Edit Last Queued Message"; if (command === "thread.copyReference") return "Pull Request: Copy Link or Thread ID"; diff --git a/apps/web/src/dictation/microphoneCapture.ts b/apps/web/src/dictation/microphoneCapture.ts new file mode 100644 index 0000000000000..7f606f67af41d --- /dev/null +++ b/apps/web/src/dictation/microphoneCapture.ts @@ -0,0 +1,107 @@ +// Records the microphone as 16 kHz mono 16-bit PCM for server-side dictation. + +export class MicrophoneUnavailableError extends Error { + constructor( + message: string, + readonly kind: "insecure-context" | "denied" | "no-device" | "failed", + ) { + super(message); + } +} + +export interface MicrophoneCapture { + /** PCM recorded since the previous call, as base64 of 16-bit little-endian samples. */ + readonly takePcm: () => string; + readonly stop: () => void; +} + +const WORKLET_URL = "dictation-capture-worklet.js"; + +export async function startMicrophoneCapture(): Promise { + if (!navigator.mediaDevices?.getUserMedia) { + // Browsers hand out the microphone only to https:// and localhost pages. + throw new MicrophoneUnavailableError( + "Dictation needs a secure connection (HTTPS or localhost).", + "insecure-context", + ); + } + let stream: MediaStream; + try { + stream = await navigator.mediaDevices.getUserMedia({ + audio: { + channelCount: 1, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); + } catch (error) { + const name = error instanceof DOMException ? error.name : ""; + if (name === "NotAllowedError" || name === "SecurityError") { + throw new MicrophoneUnavailableError("Microphone access was denied.", "denied"); + } + if (name === "NotFoundError" || name === "OverconstrainedError") { + throw new MicrophoneUnavailableError("No microphone found.", "no-device"); + } + throw new MicrophoneUnavailableError( + `The microphone cannot be opened: ${error instanceof Error ? error.message : String(error)}`, + "failed", + ); + } + const context = new AudioContext(); + const blocks: Array = []; + try { + await context.audioWorklet.addModule(new URL(WORKLET_URL, document.baseURI).href); + const source = context.createMediaStreamSource(stream); + const node = new AudioWorkletNode(context, "dictation-capture"); + node.port.addEventListener("message", (event: MessageEvent) => { + blocks.push(event.data); + }); + // addEventListener, unlike onmessage, does not start the port by itself. + node.port.start(); + source.connect(node); + // Silent output: the graph only runs nodes that lead to the speakers. + const mute = context.createGain(); + mute.gain.value = 0; + node.connect(mute).connect(context.destination); + await context.resume(); + } catch (error) { + for (const track of stream.getTracks()) track.stop(); + void context.close(); + throw new MicrophoneUnavailableError( + `Recording cannot start: ${error instanceof Error ? error.message : String(error)}`, + "failed", + ); + } + + return { + takePcm: () => { + const taken = blocks.splice(0, blocks.length); + return encodePcm16(taken); + }, + stop: () => { + for (const track of stream.getTracks()) track.stop(); + void context.close(); + }, + }; +} + +export function encodePcm16(blocks: ReadonlyArray): string { + let length = 0; + for (const block of blocks) length += block.length; + const bytes = new Uint8Array(length * 2); + const view = new DataView(bytes.buffer); + let offset = 0; + for (const block of blocks) { + for (const sample of block) { + const clamped = Math.max(-1, Math.min(1, sample)); + view.setInt16(offset, clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff, true); + offset += 2; + } + } + let binary = ""; + for (let index = 0; index < bytes.length; index += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(index, index + 0x8000)); + } + return btoa(binary); +} diff --git a/apps/web/src/dictation/useDictation.ts b/apps/web/src/dictation/useDictation.ts new file mode 100644 index 0000000000000..5537f7c274e94 --- /dev/null +++ b/apps/web/src/dictation/useDictation.ts @@ -0,0 +1,285 @@ +import { squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; +import type { EnvironmentId } from "@t3tools/contracts"; +import { useEffect, useRef, useState } from "react"; + +import { randomUUID } from "~/lib/utils"; +import { serverEnvironment } from "../state/server"; +import { useAtomCommand } from "../state/use-atom-command"; +import { + MicrophoneUnavailableError, + startMicrophoneCapture, + type MicrophoneCapture, +} from "./microphoneCapture"; + +// Dictation into the composer: the microphone is recorded here and streamed to the server, +// whose local speech model writes the transcript. On request (the structured shortcut, or +// "Reinschrift" spoken at the end) a small model rewrites it into a structured prompt first. + +const SEND_INTERVAL_MS = 500; + +export type DictationPhase = "idle" | "starting" | "recording" | "transcribing" | "structuring"; + +export interface DictationNotice { + readonly text: string; + readonly tone: "info" | "warning" | "error" | "success"; +} + +export interface DictationState { + readonly phase: DictationPhase; + /** Live transcript of the last seconds while recording. */ + readonly preview: string; + /** The running recording will be rewritten into a structured prompt. */ + readonly structured: boolean; + readonly notice: DictationNotice | null; +} + +interface Recording { + readonly sessionId: string; + readonly capture: MicrophoneCapture; + readonly timer: ReturnType; + /** Appends in flight; the finish waits for the last one. */ + sending: Promise; + failed: string | null; +} + +const IDLE: DictationState = { phase: "idle", preview: "", structured: false, notice: null }; + +const failureText = (result: { readonly _tag: "Failure" }) => { + const error = squashAtomCommandFailure(result as never) as unknown; + return error instanceof Error ? error.message : String(error); +}; + +export function useDictation(input: { + readonly environmentId: EnvironmentId | null; + /** Puts the text into the composer; false when the composer cannot take it right now. */ + readonly insert: (text: string) => boolean; +}) { + const append = useAtomCommand(serverEnvironment.dictationAppend, { reportFailure: false }); + const finish = useAtomCommand(serverEnvironment.dictationFinish, { reportFailure: false }); + const cancelSession = useAtomCommand(serverEnvironment.dictationCancel, { reportFailure: false }); + const structure = useAtomCommand(serverEnvironment.dictationStructure, { reportFailure: false }); + const [state, setRenderedState] = useState(IDLE); + // Callbacks read the state they act on from here, not from a render that may be stale. + const stateRef = useRef(state); + const setState = (next: DictationState | ((previous: DictationState) => DictationState)) => { + stateRef.current = typeof next === "function" ? next(stateRef.current) : next; + setRenderedState(stateRef.current); + }; + const recordingRef = useRef(null); + // Bumped whenever a step is abandoned (Esc), so its late result is ignored. + const generationRef = useRef(0); + const noticeTimerRef = useRef | null>(null); + // The plain transcript while it is being rewritten, for Esc to insert instead. + const unstructuredRef = useRef(null); + const inputRef = useRef(input); + useEffect(() => { + inputRef.current = input; + }); + + const update = (next: Partial) => { + setState((previous) => ({ ...previous, ...next })); + }; + + const notify = (notice: DictationNotice, hideAfterMs = 4000) => { + if (noticeTimerRef.current !== null) clearTimeout(noticeTimerRef.current); + update({ notice }); + noticeTimerRef.current = setTimeout(() => { + noticeTimerRef.current = null; + setState((previous) => + previous.notice === notice ? { ...previous, notice: null } : previous, + ); + }, hideAfterMs); + }; + + const endRecording = (): Recording | null => { + const recording = recordingRef.current; + recordingRef.current = null; + if (recording === null) return null; + clearInterval(recording.timer); + return recording; + }; + + const sendPcm = (recording: Recording, environmentId: EnvironmentId, pcm: string) => { + recording.sending = recording.sending.then(async (ok) => { + if (!ok) return false; + const result = await append({ + environmentId, + input: { sessionId: recording.sessionId, language: navigator.language, pcm }, + }); + if (result._tag === "Failure") { + recording.failed = failureText(result); + return false; + } + if (recordingRef.current === recording) update({ preview: result.value.preview }); + return true; + }); + return recording.sending; + }; + + const start = async (structured: boolean) => { + const environmentId = inputRef.current.environmentId; + if (environmentId === null) return; + const generation = ++generationRef.current; + update({ phase: "starting", preview: "", structured, notice: null }); + let capture: MicrophoneCapture; + try { + capture = await startMicrophoneCapture(); + } catch (error) { + setState(IDLE); + notify( + { + text: error instanceof MicrophoneUnavailableError ? error.message : String(error), + tone: "error", + }, + 6000, + ); + return; + } + if (generation !== generationRef.current) { + capture.stop(); + return; + } + const recording: Recording = { + sessionId: randomUUID(), + capture, + timer: setInterval(() => { + void sendPcm(recording, environmentId, capture.takePcm()).then((ok) => { + if (!ok && recordingRef.current === recording) { + endRecording(); + capture.stop(); + setState(IDLE); + notify({ text: recording.failed ?? "Dictation failed.", tone: "error" }, 8000); + } + }); + }, SEND_INTERVAL_MS), + sending: Promise.resolve(true), + failed: null, + }; + recordingRef.current = recording; + update({ phase: "recording" }); + }; + + const stop = async (structuredNow: boolean) => { + const environmentId = inputRef.current.environmentId; + const recording = endRecording(); + if (recording === null || environmentId === null) return; + const generation = generationRef.current; + const structured = stateRef.current.structured || structuredNow; + update({ phase: "transcribing", structured }); + const last = recording.capture.takePcm(); + recording.capture.stop(); + const sent = await sendPcm(recording, environmentId, last); + if (!sent) { + setState(IDLE); + notify({ text: recording.failed ?? "Dictation failed.", tone: "error" }, 8000); + return; + } + const result = await finish({ environmentId, input: { sessionId: recording.sessionId } }); + if (generation !== generationRef.current) return; + if (result._tag === "Failure") { + setState(IDLE); + notify({ text: failureText(result), tone: "error" }, 8000); + return; + } + const { text, reason, structureRequested } = result.value; + if (text === "") { + setState(IDLE); + notify({ + text: + reason === "no-audio" + ? "No sound from the microphone. Check the input device." + : "No speech recognized.", + tone: "warning", + }); + return; + } + if (!structured && !structureRequested) { + deliver(text, null); + return; + } + update({ phase: "structuring" }); + unstructuredRef.current = text; + const rewritten = await structure({ environmentId, input: { text } }); + if (generation !== generationRef.current) return; + unstructuredRef.current = null; + if (rewritten._tag === "Failure") { + console.warn("dictation: structuring failed", failureText(rewritten)); + deliver(text, "Structuring failed · inserted unchanged"); + return; + } + deliver(rewritten.value.text, null); + }; + + const deliver = (text: string, warning: string | null) => { + setState(IDLE); + if (inputRef.current.insert(text)) { + notify( + warning ? { text: warning, tone: "warning" } : { text: "Inserted", tone: "success" }, + 1500, + ); + return; + } + void navigator.clipboard?.writeText(text); + notify( + { + text: "The composer cannot take text right now · copied to the clipboard", + tone: "warning", + }, + 6000, + ); + }; + + /** Starts a recording, or stops the running one. `structured` asks for the rewrite. */ + const toggle = (structured = false) => { + const phase = stateRef.current.phase; + if (phase === "idle") void start(structured); + else if (phase === "recording") void stop(structured); + }; + + /** + * Esc: drops a recording, or skips the rewrite and inserts the plain transcript. + * Returns whether there was anything to cancel. + */ + const cancel = (): boolean => { + const phase = stateRef.current.phase; + if (phase === "recording" || phase === "starting") { + generationRef.current++; + const recording = endRecording(); + const environmentId = inputRef.current.environmentId; + if (recording !== null) { + recording.capture.stop(); + if (environmentId !== null) { + void cancelSession({ environmentId, input: { sessionId: recording.sessionId } }); + } + } + setState(IDLE); + notify({ text: "Cancelled", tone: "info" }, 1200); + return true; + } + if (phase === "structuring" && unstructuredRef.current !== null) { + generationRef.current++; + const text = unstructuredRef.current; + unstructuredRef.current = null; + deliver(text, "Inserted unchanged"); + return true; + } + return false; + }; + + useEffect( + () => () => { + // Leaving the composer (another thread, unmount) ends a recording without inserting it. + generationRef.current++; + const recording = recordingRef.current; + recordingRef.current = null; + if (recording !== null) { + clearInterval(recording.timer); + recording.capture.stop(); + } + if (noticeTimerRef.current !== null) clearTimeout(noticeTimerRef.current); + }, + [], + ); + + return { state, toggle, cancel }; +} diff --git a/packages/client-runtime/src/state/server.ts b/packages/client-runtime/src/state/server.ts index a38c28db62961..b86e49c25f3fd 100644 --- a/packages/client-runtime/src/state/server.ts +++ b/packages/client-runtime/src/state/server.ts @@ -1333,6 +1333,27 @@ export function createServerEnvironmentAtoms( label: "environment-data:server:check-otlp-endpoint", tag: WS_METHODS.serverCheckOtlpEndpoint, }), + // Chunks of one recording must arrive in order, and the end of it after the last chunk. + dictationAppend: createEnvironmentRpcCommand(runtime, { + label: "environment-data:dictation:append", + tag: WS_METHODS.dictationAppend, + concurrency: { + mode: "serial", + key: ({ environmentId, input }) => JSON.stringify([environmentId, input.sessionId]), + }, + }), + dictationFinish: createEnvironmentRpcCommand(runtime, { + label: "environment-data:dictation:finish", + tag: WS_METHODS.dictationFinish, + }), + dictationCancel: createEnvironmentRpcCommand(runtime, { + label: "environment-data:dictation:cancel", + tag: WS_METHODS.dictationCancel, + }), + dictationStructure: createEnvironmentRpcCommand(runtime, { + label: "environment-data:dictation:structure", + tag: WS_METHODS.dictationStructure, + }), retryResourceTelemetry: createEnvironmentRpcCommand(runtime, { label: "environment-data:server:retry-resource-telemetry", tag: WS_METHODS.serverRetryResourceTelemetry, diff --git a/packages/contracts/src/keybindings.ts b/packages/contracts/src/keybindings.ts index 07cdd6d9dc599..5da8a16437022 100644 --- a/packages/contracts/src/keybindings.ts +++ b/packages/contracts/src/keybindings.ts @@ -99,6 +99,8 @@ export const STATIC_KEYBINDING_COMMANDS = [ "composer.workspace", "composer.previousWorktree", "composer.branch", + "composer.dictate", + "composer.dictateStructured", "chat.new", "chat.newLocal", "chat.newWithoutProject", diff --git a/packages/shared/src/keybindings.ts b/packages/shared/src/keybindings.ts index b511f830f69b5..3b64d5a66b3b6 100644 --- a/packages/shared/src/keybindings.ts +++ b/packages/shared/src/keybindings.ts @@ -203,6 +203,8 @@ export const DEFAULT_KEYBINDINGS: ReadonlyArray = [ { key: "mod+shift+a", command: "composer.mode", when: "!terminalFocus" }, { key: "mod+shift+x", command: "composer.workspace", when: "!terminalFocus" }, { key: "mod+shift+g", command: "composer.branch", when: "!terminalFocus" }, + { key: "f9", command: "composer.dictate", when: "!terminalFocus" }, + { key: "f8", command: "composer.dictateStructured", when: "!terminalFocus" }, { key: "mod+shift+l", command: "composer.previousWorktree", when: "!terminalFocus" }, { key: "mod+shift+k", command: "pullRequest.copyNumber", when: "!terminalFocus" }, { key: "mod+shift+arrowup", command: "modelPicker.previousProvider", when: "modelPickerOpen" }, From f532800401d27dd2efe281134a043a0dbbeaee52 Mon Sep 17 00:00:00 2001 From: collissimo <148455189+collissimo@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:50:21 +0200 Subject: [PATCH 3/5] build(desktop): ship transcribe.cpp in the server sidecar The package stays external like node-pty, so its platform package (@transcribe-cpp/-, picked by os/cpu) lands in server.asar. Its DLLs are unpacked beside the archive, and the server points transcribe.cpp at them because the OS loader cannot read an asar. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/dictation/Dictation.ts | 25 +++++++++++++++++++++++-- scripts/lib/cli-external-packages.ts | 5 +++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/apps/server/src/dictation/Dictation.ts b/apps/server/src/dictation/Dictation.ts index 0bcb6c4ea9f09..9f17e03517b81 100644 --- a/apps/server/src/dictation/Dictation.ts +++ b/apps/server/src/dictation/Dictation.ts @@ -24,6 +24,28 @@ import { DEFAULT_STRUCTURE_PROMPT } from "./structurePrompt.ts"; import { DEFAULT_STRUCTURE_CODEWORDS, removeFillers, splitCodeword } from "./transcriptText.ts"; const IDLE_UNLOAD_MS = 15 * 60_000; +const NATIVE_LIBRARY_NAME = + process.platform === "win32" + ? "transcribe.dll" + : process.platform === "darwin" + ? "libtranscribe.dylib" + : "libtranscribe.so"; + +// transcribe-cpp dlopens a native library; it is only loaded once someone dictates. +const loadTranscribeCpp = async (path: Path.Path): Promise => { + const module = (await import("transcribe-cpp")) as TranscribeModule; + if (!process.env.TRANSCRIBE_LIBRARY) { + // In the desktop app the package sits in an asar archive, which the OS loader cannot read; + // its DLLs are unpacked beside the archive. + const directory = module.artifactDir(); + const unpacked = directory.replace(/\.asar([\\/])/, ".asar.unpacked$1"); + if (unpacked !== directory) { + process.env.TRANSCRIBE_LIBRARY = path.join(unpacked, NATIVE_LIBRARY_NAME); + } + } + return module; +}; + const STRUCTURE_TIMEOUT = "90 seconds"; const STRUCTURE_MODEL = "haiku"; const STRUCTURE_EFFORT = "medium"; @@ -78,8 +100,7 @@ const make = Effect.gen(function* () { Effect.sync( () => new DictationEngine({ - // transcribe-cpp dlopens a native library; it is only loaded once someone dictates. - loadModule: () => import("transcribe-cpp") as Promise, + loadModule: () => loadTranscribeCpp(path), resolveModel: (language) => Effect.runPromiseWith(services)( findModel({ language, configured: configuredModel, directories }), diff --git a/scripts/lib/cli-external-packages.ts b/scripts/lib/cli-external-packages.ts index 678d3d6e91519..7d6f2caca2981 100644 --- a/scripts/lib/cli-external-packages.ts +++ b/scripts/lib/cli-external-packages.ts @@ -21,6 +21,11 @@ export const CLI_RUNTIME_EXTERNAL_PREFIXES = [ "@yuuang/", "@ff-labs/", "@napi-rs/keyring", + // Dictation: transcribe.cpp finds its platform package beside it and dlopens the DLLs in it. + "transcribe-cpp", + "@transcribe-cpp/", + "koffi", + "@koromix/", "@clerk/electron-passkeys", "node-gyp-build", "node-addon-api", From d40e66dbbfe8579ef9ca728ca868e74883134305 Mon Sep 17 00:00:00 2001 From: collissimo <148455189+collissimo@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:50:47 +0200 Subject: [PATCH 4/5] docs(fork): describe the built-in dictation Co-Authored-By: Claude Opus 5.5 --- FORK.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/FORK.md b/FORK.md index 626817b8f4fbd..b0b2078be7b3b 100644 --- a/FORK.md +++ b/FORK.md @@ -45,3 +45,29 @@ versioned; install per clone: `cp fork/pre-push "$(git rev-parse --git-common-di hot upstream files (AGENTS.md, shared configs) when a separate file works. - Keep edits to existing upstream files small and isolated. - Sync often; small merges are easier than large ones. + +## Fork features + +### Dictation + +Mic button in the composer (next to "Attach files"), **F9** to start and stop, **F8** (or +shift-click, or ending with the word "Reinschrift") to get a structured prompt, **Esc** to +cancel a recording or to insert the plain transcript while it is being rewritten. Both keys +can be rebound in Settings → Keybindings ("Composer: Dictate"). + +- Speech recognition runs on the server's machine: Parakeet TDT 0.6b v3 through + [transcribe.cpp](https://github.com/handy-computer/transcribe.cpp) (`transcribe-cpp` npm + package). Nothing is downloaded; the `.gguf` model Scrawl + (`%LOCALAPPDATA%\Scrawl\models`) or Hex (`%APPDATA%\voice-control\models`) installed is used. + `T3CODE_DICTATION_MODEL=` picks another one. +- The model (~1.1 GB) loads on the first chunk of a recording and is let go after 15 minutes + without one. +- Structuring calls `claude --print --model haiku` on the server; it needs a logged-in + Claude Code. The instructions are built in; a file at `/dictation/structure-prompt.md` + replaces them and is read on every rewrite. +- The browser only hands out the microphone on https:// or localhost, so a web client on a + plain `http://` cannot dictate. Mobile keeps its own on-device dictation. +- Code: `apps/server/src/dictation/`, `apps/web/src/dictation/`, + `apps/web/src/components/chat/ComposerDictationButton.tsx`, `packages/contracts/src/dictation.ts`. + The standalone app for every other program is [collissimo/t3-voice](https://github.com/collissimo/t3-voice); + it leaves F9 to T3 Code while T3 Code is in front. From 9a6039bd2c84173a173525035de8209cdb032d87 Mon Sep 17 00:00:00 2001 From: collissimo <148455189+collissimo@users.noreply.github.com> Date: Wed, 21 Oct 2026 00:04:41 +0200 Subject: [PATCH 5/5] fix(web): load the dictation worklet from the app root Relative to document.baseURI it resolved under the current thread route and hit the SPA fallback (MIME type mismatch), so recording never started. Co-Authored-By: Claude Opus 5.5 --- apps/web/src/dictation/microphoneCapture.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/web/src/dictation/microphoneCapture.ts b/apps/web/src/dictation/microphoneCapture.ts index 7f606f67af41d..434f8ad4a4033 100644 --- a/apps/web/src/dictation/microphoneCapture.ts +++ b/apps/web/src/dictation/microphoneCapture.ts @@ -15,7 +15,8 @@ export interface MicrophoneCapture { readonly stop: () => void; } -const WORKLET_URL = "dictation-capture-worklet.js"; +// From the app root, not the current route: //x.js would hit the SPA fallback. +const WORKLET_URL = `${import.meta.env.BASE_URL.replace(/\/$/, "")}/dictation-capture-worklet.js`; export async function startMicrophoneCapture(): Promise { if (!navigator.mediaDevices?.getUserMedia) { @@ -51,7 +52,7 @@ export async function startMicrophoneCapture(): Promise { const context = new AudioContext(); const blocks: Array = []; try { - await context.audioWorklet.addModule(new URL(WORKLET_URL, document.baseURI).href); + await context.audioWorklet.addModule(WORKLET_URL); const source = context.createMediaStreamSource(stream); const node = new AudioWorkletNode(context, "dictation-capture"); node.port.addEventListener("message", (event: MessageEvent) => {