diff --git a/FORK.md b/FORK.md index ed213b325ab06..a03f4b11c4d0a 100644 --- a/FORK.md +++ b/FORK.md @@ -109,14 +109,19 @@ can be rebound in Settings → Keybindings ("Composer: Dictate"). and with the local environment turned off: it starts `t3 dictation-worker` from the server package it ships, a process that only receives the audio and reads the model (no server, agents, files or terminals). A browser transcribes on the server it is served from. Both use - Parakeet TDT 0.6b v3 through [transcribe.cpp](https://github.com/handy-computer/transcribe.cpp) (`transcribe-cpp` npm - package). Nothing is downloaded; the `.gguf` model Scrawl - (`%LOCALAPPDATA%\Scrawl\models`) or Hex (`%APPDATA%\voice-control\models`) installed is used. - `T3CODE_DICTATION_MODEL=` picks another one. - Without Scrawl or Hex, `fork/install-parakeet.ps1` downloads the model into Scrawl's folder. -- The model (~1.1 GB) loads on the first chunk of a recording and is let go after 15 minutes - without one. + package). Nothing is downloaded; the best `.gguf` found is used, from + `~/.t3/dictation/models` (`$T3CODE_HOME/dictation/models`; not `--home-dir`), Scrawl (`%LOCALAPPDATA%\Scrawl\models`) or Hex + (`%APPDATA%\voice-control\models`). Best first: Cohere Transcribe 03-2026, Canary 1B v2, + Parakeet TDT 0.6b v3 (on German speech they miss about 4%, 7% and 14% of the words). + `T3CODE_DICTATION_MODEL=` picks one. Without Scrawl or Hex, + `fork/install-parakeet.ps1` downloads Parakeet into Scrawl's folder; the others come from + [huggingface.co/handy-computer](https://huggingface.co/handy-computer) (the `Q8_0` file). +- It runs on the GPU when there is one (Vulkan; the log says `loaded … on Vulkan0`), about five + times faster than the CPU. `T3CODE_DICTATION_BACKEND=cpu` keeps it on the CPU. +- Long recordings are transcribed in pieces of at most 25 seconds, cut at pauses. +- The model (1 to 2.5 GB) loads on the first chunk of a recording and is let go after 15 + minutes without one. - Structuring calls `claude --print --model haiku` on the thread's server; it needs a logged-in Claude Code. The instructions are built in; a file at `/dictation/structure-prompt.md` replaces them and is read on every rewrite. diff --git a/apps/server/package.json b/apps/server/package.json index e8074b1e10403..142068e273c8c 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -57,6 +57,7 @@ "@t3tools/source-control-gitlab": "workspace:*", "effect": "catalog:", "jose": "catalog:", + "koffi": "3.3.3", "node-pty": "^1.2.0-beta.15", "playwright-core": "1.60.0", "proper-lockfile": "4.1.2", diff --git a/apps/server/src/dictation/DictationEngine.test.ts b/apps/server/src/dictation/DictationEngine.test.ts index af91bdfd800be..9cb5697ee707b 100644 --- a/apps/server/src/dictation/DictationEngine.test.ts +++ b/apps/server/src/dictation/DictationEngine.test.ts @@ -9,8 +9,10 @@ import { SAMPLE_RATE, decodePcm16, normalizeLanguage, + splitAtPauses, type TranscribeModule, } from "./DictationEngine.ts"; +import { dictationBackend } from "./localDictation.ts"; import { findModel, rankModels } from "./modelFiles.ts"; import { handleDictationWorkerRequest } from "./worker.ts"; @@ -18,6 +20,9 @@ const V3 = "C:/models/parakeet-tdt-0.6b-v3-Q8_0.gguf"; const UNIFIED_EN = "C:/models/parakeet-unified-en-0.6b-Q8_0.gguf"; const V2 = "C:/models/parakeet-tdt-0.6b-v2-Q8_0.gguf"; const WHISPER = "C:/models/whisper-large-v3-turbo-Q8_0.gguf"; +const WHISPER_EN = "C:/models/whisper-small.en-Q8_0.gguf"; +const COHERE = "C:/models/cohere-transcribe-03-2026-Q8_0.gguf"; +const CANARY = "C:/models/canary-1b-v2-Q8_0.gguf"; const basename = (path: string) => path.split("/").at(-1)!; describe("rankModels", () => { @@ -32,7 +37,16 @@ describe("rankModels", () => { ]); }); it("never gives German an English-only model", () => { - expect(rankModels([UNIFIED_EN, V2], "de", basename)).toEqual([]); + expect(rankModels([UNIFIED_EN, V2, WHISPER_EN], "de", basename)).toEqual([]); + }); + it("prefers Cohere, then Canary, for the languages they know", () => { + const all = [V3, UNIFIED_EN, CANARY, COHERE]; + expect(rankModels(all, "de", basename)).toEqual([COHERE, CANARY, V3]); + expect(rankModels(all, "en", basename)).toEqual([COHERE, CANARY, UNIFIED_EN, V3]); + expect(rankModels(all, "sv", basename)).toEqual([CANARY, V3]); + }); + it("puts models that detect the language first when none is given", () => { + expect(rankModels([COHERE, V3], undefined, basename)).toEqual([V3, COHERE]); }); }); @@ -80,6 +94,23 @@ describe("audio input", () => { pcm.writeInt16LE(-32768, 2); expect([...decodePcm16(pcm.toString("base64"))]).toEqual([0.5, -1]); }); + it("cuts long audio at the quietest moment, never longer than 25 seconds", () => { + const audio = speech(70); + audio.fill(0, 18 * SAMPLE_RATE, 18.5 * SAMPLE_RATE); + const pieces = splitAtPauses(audio); + const cut = pieces[0]!.length / SAMPLE_RATE; + expect(cut).toBeGreaterThan(18); + expect(cut).toBeLessThan(18.5); + expect(pieces.every((piece) => piece.length <= 25 * SAMPLE_RATE)).toBe(true); + expect(pieces.reduce((sum, piece) => sum + piece.length, 0)).toBe(audio.length); + expect(splitAtPauses(speech(25))).toHaveLength(1); + }); + it("reads the backend from the environment, auto when unset or unknown", () => { + expect(dictationBackend(" CPU ")).toBe("cpu"); + expect(dictationBackend("vulkan")).toBe("vulkan"); + expect(dictationBackend("gpu")).toBe("auto"); + expect(dictationBackend(undefined)).toBe("auto"); + }); it("normalizes language tags", () => { expect(normalizeLanguage("de-DE")).toBe("de"); expect(normalizeLanguage(" EN_us ")).toBe("en"); @@ -142,6 +173,12 @@ describe("DictationEngine", () => { expect(FakeModel.loads).toEqual(["model.gguf"]); }); + it("transcribes a long recording in pieces and joins them", async () => { + const { engine } = makeEngine(); + engine.append("long", "de", speech(40)); + expect(await engine.finish("long")).toEqual({ text: "Hallo Welt Hallo Welt" }); + }); + it("reports silence and too short recordings without asking the model", async () => { const { engine } = makeEngine(); engine.append("quiet", "de", new Float32Array(SAMPLE_RATE)); diff --git a/apps/server/src/dictation/DictationEngine.ts b/apps/server/src/dictation/DictationEngine.ts index e8254e91eebb1..ebb577245f265 100644 --- a/apps/server/src/dictation/DictationEngine.ts +++ b/apps/server/src/dictation/DictationEngine.ts @@ -2,8 +2,8 @@ // // A client streams microphone audio into a recording session. The model loads on the first // chunk, while the user is already speaking, and is let go after a stretch without -// recordings (it holds about 1.1 GB). While recording, the last seconds are transcribed again -// and again for a live preview; the final transcript is one run over the whole recording. +// recordings (it holds 1 to 2.5 GB). While recording, the last seconds are transcribed again +// and again for a live preview; the final transcript is the whole recording, cut at pauses. import type * as TranscribeCpp from "transcribe-cpp"; @@ -16,6 +16,12 @@ const SILENT_PEAK = 0.002; // length. The client shows the last few hundred characters, which hides the clipped first word. const PREVIEW_WINDOW_SECONDS = 20; const PREVIEW_PAUSE_MS = 800; +// One run over minutes of audio goes wrong: Parakeet drifts into gibberish, and encoder-decoder +// models (Cohere, Canary) stop at their token limit. The final transcript is run in pieces cut at +// the quietest moment between these bounds. +const CHUNK_MIN_SECONDS = 15; +const CHUNK_MAX_SECONDS = 25; +const PAUSE_FRAME = SAMPLE_RATE / 10; const MAX_RECORDING_SECONDS = 15 * 60; // A client that vanished mid-recording leaves its session behind. const ABANDONED_SESSION_MS = 2 * 60_000; @@ -28,7 +34,7 @@ export interface DictationEngineOptions { readonly resolveModel: (language: string | undefined) => Promise; /** Let go of the model after this long without a recording; 0 keeps it. */ readonly idleUnloadMs: number; - /** transcribe.cpp backend. The CPU is the default: Vulkan returns empty text on some GPUs. */ + /** transcribe.cpp backend; "auto" picks the GPU when there is one. */ readonly backend?: TranscribeCpp.Backend; readonly now?: () => number; readonly log?: (message: string) => void; @@ -60,6 +66,32 @@ export function normalizeLanguage(language: string | undefined): string | undefi return code ? code : undefined; } +/** Cuts audio into pieces of at most CHUNK_MAX_SECONDS, each ending in its quietest stretch. */ +export function splitAtPauses(audio: Float32Array): ReadonlyArray { + const min = CHUNK_MIN_SECONDS * SAMPLE_RATE; + const max = CHUNK_MAX_SECONDS * SAMPLE_RATE; + const pieces: Array = []; + let start = 0; + while (audio.length - start > max) { + let cut = start + max; + let quietest = Infinity; + for (let frame = start + min; frame + PAUSE_FRAME <= start + max; frame += PAUSE_FRAME / 2) { + let energy = 0; + for (let index = frame; index < frame + PAUSE_FRAME; index++) { + energy += audio[index]! * audio[index]!; + } + if (energy < quietest) { + quietest = energy; + cut = frame + PAUSE_FRAME / 2; + } + } + pieces.push(audio.subarray(start, cut)); + start = cut; + } + pieces.push(audio.subarray(start)); + return pieces; +} + export function decodePcm16(base64: string): Float32Array { const bytes = Buffer.from(base64, "base64"); const count = Math.floor(bytes.length / 2); @@ -140,14 +172,20 @@ export class DictationEngine { this.#previewAbort?.abort(); const started = this.#now(); const text = await this.#exclusive(async () => { - try { - const result = await loaded.session.run(audio, this.#runOptions(loaded, session.language)); - return result.text; - } catch (error) { - const module = await this.#loadModule(); - if (error instanceof module.InputTooShort) return ""; - throw error; + const texts: Array = []; + for (const piece of splitAtPauses(audio)) { + try { + const result = await loaded.session.run( + piece, + this.#runOptions(loaded, session.language), + ); + texts.push(result.text); + } catch (error) { + const module = await this.#loadModule(); + if (!(error instanceof module.InputTooShort)) throw error; + } } + return texts.join(" "); }); this.#lastUsedAt = this.#now(); const cleaned = text.split(/\s+/).filter(Boolean).join(" "); @@ -256,7 +294,7 @@ export class DictationEngine { const module = await this.#loadModule(); const started = this.#now(); const model = await module.TranscribeModel.load(path, { - backend: this.#options.backend ?? "cpu", + backend: this.#options.backend ?? "auto", }); const loaded: LoadedModel = { path, diff --git a/apps/server/src/dictation/localDictation.ts b/apps/server/src/dictation/localDictation.ts index 989e0e7d0718d..9d7ed2b621411 100644 --- a/apps/server/src/dictation/localDictation.ts +++ b/apps/server/src/dictation/localDictation.ts @@ -3,6 +3,7 @@ // transcribe.cpp, the engine with its model lookup, and turning a transcript into a result. import type { DictationFinishResult } from "@t3tools/contracts"; +import type * as TranscribeCpp from "transcribe-cpp"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Effect from "effect/Effect"; import * as Path from "effect/Path"; @@ -18,15 +19,40 @@ import { DEFAULT_STRUCTURE_CODEWORDS, removeFillers, splitCodeword } from "./tra const IDLE_UNLOAD_MS = 15 * 60_000; export const HOUSEKEEPING_INTERVAL_MS = 30_000; +// oxlint-disable-next-line t3code/no-global-process-runtime -- Also runs in the desktop worker, which has no Effect runtime. +const PLATFORM = process.platform; const NATIVE_LIBRARY_NAME = - process.platform === "win32" + PLATFORM === "win32" ? "transcribe.dll" - : process.platform === "darwin" + : PLATFORM === "darwin" ? "libtranscribe.dylib" : "libtranscribe.so"; +const BACKENDS = ["auto", "cpu", "cpu_accel", "cuda", "rocm", "vulkan", "metal"] as const; + +/** T3CODE_DICTATION_BACKEND, e.g. "cpu" when the GPU is busy or misbehaves; "auto" by default. */ +export function dictationBackend(value: string | undefined): TranscribeCpp.Backend { + const wanted = value?.trim().toLowerCase(); + return BACKENDS.find((backend) => backend === wanted) ?? "auto"; +} + +// ggml-vulkan's cooperative-matrix path returns empty transcripts on AMD RDNA4 (RX 9070 XT); +// without it the GPU is still about five times faster than the CPU. ggml reads the switch with +// getenv when the backend starts. On Windows that is the C runtime's copy of the environment, +// which `process.env` does not update, so it is set through the C runtime itself. +const disableVulkanCoopmat = async (): Promise => { + const name = "GGML_VK_DISABLE_COOPMAT"; + if (process.env[name] !== undefined) return; + if (PLATFORM === "win32") { + const koffi = (await import("koffi")).default; + koffi.load("ucrtbase.dll").func("int _putenv_s(str, str)")(name, "1"); + } + process.env[name] = "1"; +}; + // transcribe-cpp dlopens a native library; it is only loaded once someone dictates. const loadTranscribeCpp = async (): Promise => { + await disableVulkanCoopmat(); const module = (await import("transcribe-cpp")) as TranscribeModule; if (!process.env.TRANSCRIBE_LIBRARY) { // In the desktop app the package sits in an asar archive, which the OS loader cannot read; @@ -54,6 +80,7 @@ export function makeLocalDictationEngine(log: (message: string) => void): Dictat resolveModel: (language) => Effect.runPromise(resolveModel(language).pipe(Effect.provide(NodeServices.layer))), idleUnloadMs: IDLE_UNLOAD_MS, + backend: dictationBackend(process.env.T3CODE_DICTATION_BACKEND), log, }); } diff --git a/apps/server/src/dictation/modelFiles.ts b/apps/server/src/dictation/modelFiles.ts index f70c9a95bb441..455d31b1410ef 100644 --- a/apps/server/src/dictation/modelFiles.ts +++ b/apps/server/src/dictation/modelFiles.ts @@ -4,55 +4,70 @@ import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; // The speech model is never downloaded here. Hex (the Windows build of voice-control) and -// Scrawl install Parakeet as a .gguf file; dictation picks it up from their folders. +// Scrawl install Parakeet as a .gguf file; dictation picks it up from their folders. Any other +// model transcribe.cpp runs goes into ~/.t3/dictation/models ($T3CODE_HOME/dictation/models). export function modelDirectories(env: NodeJS.ProcessEnv, path: Path.Path): ReadonlyArray { const dirs: Array = []; + const home = env.USERPROFILE || env.HOME; + const t3Home = env.T3CODE_HOME?.trim() || (home ? path.join(home, ".t3") : undefined); + if (t3Home) dirs.push(path.join(t3Home, "dictation", "models")); if (env.APPDATA) dirs.push(path.join(env.APPDATA, "voice-control", "models")); if (env.LOCALAPPDATA) dirs.push(path.join(env.LOCALAPPDATA, "Scrawl", "models")); return dirs; } -const nameParts = (name: string) => name.toLowerCase().replace(/[_.]/g, "-").split("-"); -const isEnglishOnly = (name: string) => { - const parts = nameParts(name); - return parts.includes("en") || parts.includes("v2"); -}; -const isMultilingual = (name: string) => { - const lowered = name.toLowerCase(); - return !isEnglishOnly(name) && (lowered.includes("v3") || lowered.includes("whisper")); -}; +const EUROPEAN = "bg hr cs da nl en et fi fr de el hu it lv lt mt pl pt ro sk sl es sv ru uk"; /** - * The models that can transcribe `language`, best first. English prefers Parakeet Unified - * English, then any Parakeet. Every other language needs a model whose name says it is - * multilingual (Parakeet v3 covers 25 European languages, or Whisper); Parakeet Unified English - * and Parakeet v2 never get German speech. Without a language, multilingual models come first. + * Models known by file name, best first, with the languages they transcribe. On German speech + * (Multilingual LibriSpeech) Cohere Transcribe misses 4% of the words, Canary 1B v2 7%, Parakeet + * v3 14%. Cohere and Canary cannot detect the language; the client always sends one. + */ +const KNOWN_MODELS: ReadonlyArray<{ + readonly matches: (name: string) => boolean; + readonly languages: ReadonlyArray | "any"; +}> = [ + { + matches: (name) => name.startsWith("cohere-transcribe") && !name.includes("arabic"), + languages: "en fr de es it pt nl pl el ar ja zh vi ko".split(" "), + }, + { matches: (name) => name.startsWith("canary-1b-v2"), languages: EUROPEAN.split(" ") }, + { matches: (name) => name.startsWith("parakeet-unified-en"), languages: ["en"] }, + { + matches: (name) => name.startsWith("parakeet") && name.includes("v3"), + languages: EUROPEAN.split(" "), + }, + { matches: (name) => name.startsWith("parakeet"), languages: ["en"] }, + { matches: (name) => name.includes("whisper") && /[.-]en[.-]/.test(name), languages: ["en"] }, + { matches: (name) => name.includes("whisper"), languages: "any" }, +]; + +/** + * The models that can transcribe `language`, best first. A model not in KNOWN_MODELS only gets + * English. Without a language, models that detect it (Parakeet v3, Whisper) come first. */ export function rankModels( paths: ReadonlyArray, language: string | undefined, basename: (path: string) => string, ): ReadonlyArray { - const english = language?.toLowerCase().startsWith("en") ?? false; - const rank = (path: string): ReadonlyArray => { + const code = language?.toLowerCase().split(/[-_]/)[0] || undefined; + const rank = (path: string): number | undefined => { const name = basename(path).toLowerCase(); - return [ - !name.startsWith("parakeet"), - !(english && name.includes("unified-en")), - !(!english && name.includes("v3")), - name, - ]; - }; - const compare = (left: string, right: string) => { - const a = rank(left); - const b = rank(right); - for (let index = 0; index < a.length; index++) { - if (a[index]! < b[index]!) return -1; - if (a[index]! > b[index]!) return 1; + const known = KNOWN_MODELS.findIndex((model) => model.matches(name)); + if (known === -1) return code === undefined || code === "en" ? KNOWN_MODELS.length : undefined; + const { languages } = KNOWN_MODELS[known]!; + if (code === undefined) { + const detects = languages === "any" || name.includes("v3"); + return detects ? known : known + KNOWN_MODELS.length; } - return 0; + return languages === "any" || languages.includes(code) ? known : undefined; }; - return paths.filter((path) => english || isMultilingual(basename(path))).toSorted(compare); + return paths + .map((path) => ({ path, rank: rank(path) })) + .filter((entry): entry is { path: string; rank: number } => entry.rank !== undefined) + .toSorted((left, right) => left.rank - right.rank || left.path.localeCompare(right.path)) + .map((entry) => entry.path); } /** The model file to load: `configured` when set, else the best one Hex or Scrawl installed. */ @@ -83,8 +98,9 @@ export const findModel = Effect.fn("dictation.findModel")(function* (input: { if (best === undefined) { return yield* new DictationError({ reason: - `No Parakeet model${input.language ? ` for "${input.language}"` : ""} found. ` + - `Install Parakeet v3 with Scrawl or Hex, or set T3CODE_DICTATION_MODEL. Searched: ` + + `No speech model${input.language ? ` for "${input.language}"` : ""} found. ` + + `Install Parakeet v3 with Scrawl or Hex, put a transcribe.cpp .gguf into the first ` + + `folder below, or set T3CODE_DICTATION_MODEL. Searched: ` + (input.directories.join(", ") || "nothing"), }); } diff --git a/apps/web/src/dictation/dictationLanguages.tsx b/apps/web/src/dictation/dictationLanguages.tsx index 0f09dabd4da1e..b5454590ac412 100644 --- a/apps/web/src/dictation/dictationLanguages.tsx +++ b/apps/web/src/dictation/dictationLanguages.tsx @@ -1,6 +1,6 @@ import type { ReactNode } from "react"; -// Languages Parakeet v3 transcribes, each with the flag of its country. Drawn as SVG because +// Languages the dictation models transcribe, each with the flag of its country. Drawn as SVG because // Windows renders flag emoji as two letters. const stripes = (direction: "row" | "column", colors: ReadonlyArray) => colors.map((color, index) => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 646151837082a..7a49c47a83a1c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -897,6 +897,9 @@ importers: jose: specifier: 'catalog:' version: 6.2.2 + koffi: + specifier: 3.3.3 + version: 3.3.3 node-pty: specifier: ^1.2.0-beta.15 version: 1.2.0-beta.15(patch_hash=f1e680afec85f650699ea756f91f469844ec4c976f4f168492a45c6b3f67d651)