feat(dictation): transcribe on the GPU, in pieces, with the best installed model

FORK.md

@@ -109,14 +109,19 @@ can be rebound in Settings → Keybindings ("Composer: Dictate").
   and with the local environment turned off: it starts `t3 dictation-worker` from the server
   package it ships, a process that only receives the audio and reads the model (no server,
   agents, files or terminals). A browser transcribes on the server it is served from. Both use
-  Parakeet TDT 0.6b v3 through
   [transcribe.cpp](https://github.com/handy-computer/transcribe.cpp) (`transcribe-cpp` npm
-  package). Nothing is downloaded; the `.gguf` model Scrawl
-  (`%LOCALAPPDATA%\Scrawl\models`) or Hex (`%APPDATA%\voice-control\models`) installed is used.
-  `T3CODE_DICTATION_MODEL=<path to .gguf>` picks another one.
-  Without Scrawl or Hex, `fork/install-parakeet.ps1` downloads the model into Scrawl's folder.
-- The model (~1.1 GB) loads on the first chunk of a recording and is let go after 15 minutes
-  without one.
+  package). Nothing is downloaded; the best `.gguf` found is used, from
+  `~/.t3/dictation/models` (`$T3CODE_HOME/dictation/models`; not `--home-dir`), Scrawl (`%LOCALAPPDATA%\Scrawl\models`) or Hex
+  (`%APPDATA%\voice-control\models`). Best first: Cohere Transcribe 03-2026, Canary 1B v2,
+  Parakeet TDT 0.6b v3 (on German speech they miss about 4%, 7% and 14% of the words).
+  `T3CODE_DICTATION_MODEL=<path to .gguf>` picks one. Without Scrawl or Hex,
+  `fork/install-parakeet.ps1` downloads Parakeet into Scrawl's folder; the others come from
+  [huggingface.co/handy-computer](https://huggingface.co/handy-computer) (the `Q8_0` file).
+- It runs on the GPU when there is one (Vulkan; the log says `loaded … on Vulkan0`), about five
+  times faster than the CPU. `T3CODE_DICTATION_BACKEND=cpu` keeps it on the CPU.
+- Long recordings are transcribed in pieces of at most 25 seconds, cut at pauses.
+- The model (1 to 2.5 GB) loads on the first chunk of a recording and is let go after 15
+  minutes without one.
 - Structuring calls `claude --print --model haiku` on the thread's server; it needs a logged-in
   Claude Code. The instructions are built in; a file at `<T3 home>/dictation/structure-prompt.md`
   replaces them and is read on every rewrite.

apps/server/package.json

@@ -57,6 +57,7 @@
     "@t3tools/source-control-gitlab": "workspace:*",
     "effect": "catalog:",
     "jose": "catalog:",
+    "koffi": "3.3.3",
     "node-pty": "^1.2.0-beta.15",
     "playwright-core": "1.60.0",
     "proper-lockfile": "4.1.2",

apps/server/src/dictation/DictationEngine.test.ts

@@ -9,15 +9,20 @@ import {
   SAMPLE_RATE,
   decodePcm16,
   normalizeLanguage,
+  splitAtPauses,
   type TranscribeModule,
 } from "./DictationEngine.ts";
+import { dictationBackend } from "./localDictation.ts";
 import { findModel, rankModels } from "./modelFiles.ts";
 import { handleDictationWorkerRequest } from "./worker.ts";
 
 const V3 = "C:/models/parakeet-tdt-0.6b-v3-Q8_0.gguf";
 const UNIFIED_EN = "C:/models/parakeet-unified-en-0.6b-Q8_0.gguf";
 const V2 = "C:/models/parakeet-tdt-0.6b-v2-Q8_0.gguf";
 const WHISPER = "C:/models/whisper-large-v3-turbo-Q8_0.gguf";
+const WHISPER_EN = "C:/models/whisper-small.en-Q8_0.gguf";
+const COHERE = "C:/models/cohere-transcribe-03-2026-Q8_0.gguf";
+const CANARY = "C:/models/canary-1b-v2-Q8_0.gguf";
 const basename = (path: string) => path.split("/").at(-1)!;
 
 describe("rankModels", () => {
@@ -32,7 +37,16 @@ describe("rankModels", () => {
     ]);
   });
   it("never gives German an English-only model", () => {
-    expect(rankModels([UNIFIED_EN, V2], "de", basename)).toEqual([]);
+    expect(rankModels([UNIFIED_EN, V2, WHISPER_EN], "de", basename)).toEqual([]);
+  });
+  it("prefers Cohere, then Canary, for the languages they know", () => {
+    const all = [V3, UNIFIED_EN, CANARY, COHERE];
+    expect(rankModels(all, "de", basename)).toEqual([COHERE, CANARY, V3]);
+    expect(rankModels(all, "en", basename)).toEqual([COHERE, CANARY, UNIFIED_EN, V3]);
+    expect(rankModels(all, "sv", basename)).toEqual([CANARY, V3]);
+  });
+  it("puts models that detect the language first when none is given", () => {
+    expect(rankModels([COHERE, V3], undefined, basename)).toEqual([V3, COHERE]);
   });
 });
 
@@ -80,6 +94,23 @@ describe("audio input", () => {
     pcm.writeInt16LE(-32768, 2);
     expect([...decodePcm16(pcm.toString("base64"))]).toEqual([0.5, -1]);
   });
+  it("cuts long audio at the quietest moment, never longer than 25 seconds", () => {
+    const audio = speech(70);
+    audio.fill(0, 18 * SAMPLE_RATE, 18.5 * SAMPLE_RATE);
+    const pieces = splitAtPauses(audio);
+    const cut = pieces[0]!.length / SAMPLE_RATE;
+    expect(cut).toBeGreaterThan(18);
+    expect(cut).toBeLessThan(18.5);
+    expect(pieces.every((piece) => piece.length <= 25 * SAMPLE_RATE)).toBe(true);
+    expect(pieces.reduce((sum, piece) => sum + piece.length, 0)).toBe(audio.length);
+    expect(splitAtPauses(speech(25))).toHaveLength(1);
+  });
+  it("reads the backend from the environment, auto when unset or unknown", () => {
+    expect(dictationBackend(" CPU ")).toBe("cpu");
+    expect(dictationBackend("vulkan")).toBe("vulkan");
+    expect(dictationBackend("gpu")).toBe("auto");
+    expect(dictationBackend(undefined)).toBe("auto");
+  });
   it("normalizes language tags", () => {
     expect(normalizeLanguage("de-DE")).toBe("de");
     expect(normalizeLanguage(" EN_us ")).toBe("en");
@@ -142,6 +173,12 @@ describe("DictationEngine", () => {
     expect(FakeModel.loads).toEqual(["model.gguf"]);
   });
 
+  it("transcribes a long recording in pieces and joins them", async () => {
+    const { engine } = makeEngine();
+    engine.append("long", "de", speech(40));
+    expect(await engine.finish("long")).toEqual({ text: "Hallo Welt Hallo Welt" });
+  });
+
   it("reports silence and too short recordings without asking the model", async () => {
     const { engine } = makeEngine();
     engine.append("quiet", "de", new Float32Array(SAMPLE_RATE));

apps/server/src/dictation/DictationEngine.ts

@@ -2,8 +2,8 @@
 //
 // A client streams microphone audio into a recording session. The model loads on the first
 // chunk, while the user is already speaking, and is let go after a stretch without
-// recordings (it holds about 1.1 GB). While recording, the last seconds are transcribed again
-// and again for a live preview; the final transcript is one run over the whole recording.
+// recordings (it holds 1 to 2.5 GB). While recording, the last seconds are transcribed again
+// and again for a live preview; the final transcript is the whole recording, cut at pauses.
 
 import type * as TranscribeCpp from "transcribe-cpp";
 
@@ -16,6 +16,12 @@ const SILENT_PEAK = 0.002;
 // length. The client shows the last few hundred characters, which hides the clipped first word.
 const PREVIEW_WINDOW_SECONDS = 20;
 const PREVIEW_PAUSE_MS = 800;
+// One run over minutes of audio goes wrong: Parakeet drifts into gibberish, and encoder-decoder
+// models (Cohere, Canary) stop at their token limit. The final transcript is run in pieces cut at
+// the quietest moment between these bounds.
+const CHUNK_MIN_SECONDS = 15;
+const CHUNK_MAX_SECONDS = 25;
+const PAUSE_FRAME = SAMPLE_RATE / 10;
 const MAX_RECORDING_SECONDS = 15 * 60;
 // A client that vanished mid-recording leaves its session behind.
 const ABANDONED_SESSION_MS = 2 * 60_000;
@@ -28,7 +34,7 @@ export interface DictationEngineOptions {
   readonly resolveModel: (language: string | undefined) => Promise<string>;
   /** Let go of the model after this long without a recording; 0 keeps it. */
   readonly idleUnloadMs: number;
-  /** transcribe.cpp backend. The CPU is the default: Vulkan returns empty text on some GPUs. */
+  /** transcribe.cpp backend; "auto" picks the GPU when there is one. */
   readonly backend?: TranscribeCpp.Backend;
   readonly now?: () => number;
   readonly log?: (message: string) => void;
@@ -60,6 +66,32 @@ export function normalizeLanguage(language: string | undefined): string | undefi
   return code ? code : undefined;
 }
 
+/** Cuts audio into pieces of at most CHUNK_MAX_SECONDS, each ending in its quietest stretch. */
+export function splitAtPauses(audio: Float32Array): ReadonlyArray<Float32Array> {
+  const min = CHUNK_MIN_SECONDS * SAMPLE_RATE;
+  const max = CHUNK_MAX_SECONDS * SAMPLE_RATE;
+  const pieces: Array<Float32Array> = [];
+  let start = 0;
+  while (audio.length - start > max) {
+    let cut = start + max;
+    let quietest = Infinity;
+    for (let frame = start + min; frame + PAUSE_FRAME <= start + max; frame += PAUSE_FRAME / 2) {
+      let energy = 0;
+      for (let index = frame; index < frame + PAUSE_FRAME; index++) {
+        energy += audio[index]! * audio[index]!;
+      }
+      if (energy < quietest) {
+        quietest = energy;
+        cut = frame + PAUSE_FRAME / 2;
+      }
+    }
+    pieces.push(audio.subarray(start, cut));
+    start = cut;
+  }
+  pieces.push(audio.subarray(start));
+  return pieces;
+}
+
 export function decodePcm16(base64: string): Float32Array {
   const bytes = Buffer.from(base64, "base64");
   const count = Math.floor(bytes.length / 2);
@@ -140,14 +172,20 @@ export class DictationEngine {
     this.#previewAbort?.abort();
     const started = this.#now();
     const text = await this.#exclusive(async () => {
-      try {
-        const result = await loaded.session.run(audio, this.#runOptions(loaded, session.language));
-        return result.text;
-      } catch (error) {
-        const module = await this.#loadModule();
-        if (error instanceof module.InputTooShort) return "";
-        throw error;
+      const texts: Array<string> = [];
+      for (const piece of splitAtPauses(audio)) {
+        try {
+          const result = await loaded.session.run(
+            piece,
+            this.#runOptions(loaded, session.language),
+          );
+          texts.push(result.text);
+        } catch (error) {
+          const module = await this.#loadModule();
+          if (!(error instanceof module.InputTooShort)) throw error;
+        }
       }
+      return texts.join(" ");
     });
     this.#lastUsedAt = this.#now();
     const cleaned = text.split(/\s+/).filter(Boolean).join(" ");
@@ -256,7 +294,7 @@ export class DictationEngine {
       const module = await this.#loadModule();
       const started = this.#now();
       const model = await module.TranscribeModel.load(path, {
-        backend: this.#options.backend ?? "cpu",
+        backend: this.#options.backend ?? "auto",
       });
       const loaded: LoadedModel = {
         path,

apps/server/src/dictation/localDictation.ts

@@ -3,6 +3,7 @@
 // transcribe.cpp, the engine with its model lookup, and turning a transcript into a result.
 
 import type { DictationFinishResult } from "@t3tools/contracts";
+import type * as TranscribeCpp from "transcribe-cpp";
 import * as NodeServices from "@effect/platform-node/NodeServices";
 import * as Effect from "effect/Effect";
 import * as Path from "effect/Path";
@@ -18,15 +19,40 @@ import { DEFAULT_STRUCTURE_CODEWORDS, removeFillers, splitCodeword } from "./tra
 
 const IDLE_UNLOAD_MS = 15 * 60_000;
 export const HOUSEKEEPING_INTERVAL_MS = 30_000;
+// oxlint-disable-next-line t3code/no-global-process-runtime -- Also runs in the desktop worker, which has no Effect runtime.
+const PLATFORM = process.platform;
 const NATIVE_LIBRARY_NAME =
-  process.platform === "win32"
+  PLATFORM === "win32"
     ? "transcribe.dll"
-    : process.platform === "darwin"
+    : PLATFORM === "darwin"
       ? "libtranscribe.dylib"
       : "libtranscribe.so";
 
+const BACKENDS = ["auto", "cpu", "cpu_accel", "cuda", "rocm", "vulkan", "metal"] as const;
+
+/** T3CODE_DICTATION_BACKEND, e.g. "cpu" when the GPU is busy or misbehaves; "auto" by default. */
+export function dictationBackend(value: string | undefined): TranscribeCpp.Backend {
+  const wanted = value?.trim().toLowerCase();
+  return BACKENDS.find((backend) => backend === wanted) ?? "auto";
+}
+
+// ggml-vulkan's cooperative-matrix path returns empty transcripts on AMD RDNA4 (RX 9070 XT);
+// without it the GPU is still about five times faster than the CPU. ggml reads the switch with
+// getenv when the backend starts. On Windows that is the C runtime's copy of the environment,
+// which `process.env` does not update, so it is set through the C runtime itself.
+const disableVulkanCoopmat = async (): Promise<void> => {
+  const name = "GGML_VK_DISABLE_COOPMAT";
+  if (process.env[name] !== undefined) return;
+  if (PLATFORM === "win32") {
+    const koffi = (await import("koffi")).default;
+    koffi.load("ucrtbase.dll").func("int _putenv_s(str, str)")(name, "1");
+  }
+  process.env[name] = "1";
+};
+
 // transcribe-cpp dlopens a native library; it is only loaded once someone dictates.
 const loadTranscribeCpp = async (): Promise<TranscribeModule> => {
+  await disableVulkanCoopmat();
   const module = (await import("transcribe-cpp")) as TranscribeModule;
   if (!process.env.TRANSCRIBE_LIBRARY) {
     // In the desktop app the package sits in an asar archive, which the OS loader cannot read;
@@ -54,6 +80,7 @@ export function makeLocalDictationEngine(log: (message: string) => void): Dictat
     resolveModel: (language) =>
       Effect.runPromise(resolveModel(language).pipe(Effect.provide(NodeServices.layer))),
     idleUnloadMs: IDLE_UNLOAD_MS,
+    backend: dictationBackend(process.env.T3CODE_DICTATION_BACKEND),
     log,
   });
 }

apps/server/src/dictation/modelFiles.ts

@@ -4,55 +4,70 @@ import * as FileSystem from "effect/FileSystem";
 import * as Path from "effect/Path";
 
 // The speech model is never downloaded here. Hex (the Windows build of voice-control) and
-// Scrawl install Parakeet as a .gguf file; dictation picks it up from their folders.
+// Scrawl install Parakeet as a .gguf file; dictation picks it up from their folders. Any other
+// model transcribe.cpp runs goes into ~/.t3/dictation/models ($T3CODE_HOME/dictation/models).
 export function modelDirectories(env: NodeJS.ProcessEnv, path: Path.Path): ReadonlyArray<string> {
   const dirs: Array<string> = [];
+  const home = env.USERPROFILE || env.HOME;
+  const t3Home = env.T3CODE_HOME?.trim() || (home ? path.join(home, ".t3") : undefined);
+  if (t3Home) dirs.push(path.join(t3Home, "dictation", "models"));
   if (env.APPDATA) dirs.push(path.join(env.APPDATA, "voice-control", "models"));
   if (env.LOCALAPPDATA) dirs.push(path.join(env.LOCALAPPDATA, "Scrawl", "models"));
   return dirs;
 }
 
-const nameParts = (name: string) => name.toLowerCase().replace(/[_.]/g, "-").split("-");
-const isEnglishOnly = (name: string) => {
-  const parts = nameParts(name);
-  return parts.includes("en") || parts.includes("v2");
-};
-const isMultilingual = (name: string) => {
-  const lowered = name.toLowerCase();
-  return !isEnglishOnly(name) && (lowered.includes("v3") || lowered.includes("whisper"));
-};
+const EUROPEAN = "bg hr cs da nl en et fi fr de el hu it lv lt mt pl pt ro sk sl es sv ru uk";
 
 /**
- * The models that can transcribe `language`, best first. English prefers Parakeet Unified
- * English, then any Parakeet. Every other language needs a model whose name says it is
- * multilingual (Parakeet v3 covers 25 European languages, or Whisper); Parakeet Unified English
- * and Parakeet v2 never get German speech. Without a language, multilingual models come first.
+ * Models known by file name, best first, with the languages they transcribe. On German speech
+ * (Multilingual LibriSpeech) Cohere Transcribe misses 4% of the words, Canary 1B v2 7%, Parakeet
+ * v3 14%. Cohere and Canary cannot detect the language; the client always sends one.
+ */
+const KNOWN_MODELS: ReadonlyArray<{
+  readonly matches: (name: string) => boolean;
+  readonly languages: ReadonlyArray<string> | "any";
+}> = [
+  {
+    matches: (name) => name.startsWith("cohere-transcribe") && !name.includes("arabic"),
+    languages: "en fr de es it pt nl pl el ar ja zh vi ko".split(" "),
+  },
+  { matches: (name) => name.startsWith("canary-1b-v2"), languages: EUROPEAN.split(" ") },
+  { matches: (name) => name.startsWith("parakeet-unified-en"), languages: ["en"] },
+  {
+    matches: (name) => name.startsWith("parakeet") && name.includes("v3"),
+    languages: EUROPEAN.split(" "),
+  },
+  { matches: (name) => name.startsWith("parakeet"), languages: ["en"] },
+  { matches: (name) => name.includes("whisper") && /[.-]en[.-]/.test(name), languages: ["en"] },
+  { matches: (name) => name.includes("whisper"), languages: "any" },
+];
+
+/**
+ * The models that can transcribe `language`, best first. A model not in KNOWN_MODELS only gets
+ * English. Without a language, models that detect it (Parakeet v3, Whisper) come first.
  */
 export function rankModels(
   paths: ReadonlyArray<string>,
   language: string | undefined,
   basename: (path: string) => string,
 ): ReadonlyArray<string> {
-  const english = language?.toLowerCase().startsWith("en") ?? false;
-  const rank = (path: string): ReadonlyArray<boolean | string> => {
+  const code = language?.toLowerCase().split(/[-_]/)[0] || undefined;
+  const rank = (path: string): number | undefined => {
     const name = basename(path).toLowerCase();
-    return [
-      !name.startsWith("parakeet"),
-      !(english && name.includes("unified-en")),
-      !(!english && name.includes("v3")),
-      name,
-    ];
-  };
-  const compare = (left: string, right: string) => {
-    const a = rank(left);
-    const b = rank(right);
-    for (let index = 0; index < a.length; index++) {
-      if (a[index]! < b[index]!) return -1;
-      if (a[index]! > b[index]!) return 1;
+    const known = KNOWN_MODELS.findIndex((model) => model.matches(name));
+    if (known === -1) return code === undefined || code === "en" ? KNOWN_MODELS.length : undefined;
+    const { languages } = KNOWN_MODELS[known]!;
+    if (code === undefined) {
+      const detects = languages === "any" || name.includes("v3");
+      return detects ? known : known + KNOWN_MODELS.length;
     }
-    return 0;
+    return languages === "any" || languages.includes(code) ? known : undefined;
   };
-  return paths.filter((path) => english || isMultilingual(basename(path))).toSorted(compare);
+  return paths
+    .map((path) => ({ path, rank: rank(path) }))
+    .filter((entry): entry is { path: string; rank: number } => entry.rank !== undefined)
+    .toSorted((left, right) => left.rank - right.rank || left.path.localeCompare(right.path))
+    .map((entry) => entry.path);
 }
 
 /** The model file to load: `configured` when set, else the best one Hex or Scrawl installed. */
@@ -83,8 +98,9 @@ export const findModel = Effect.fn("dictation.findModel")(function* (input: {
   if (best === undefined) {
     return yield* new DictationError({
       reason:
-        `No Parakeet model${input.language ? ` for "${input.language}"` : ""} found. ` +
-        `Install Parakeet v3 with Scrawl or Hex, or set T3CODE_DICTATION_MODEL. Searched: ` +
+        `No speech model${input.language ? ` for "${input.language}"` : ""} found. ` +
+        `Install Parakeet v3 with Scrawl or Hex, put a transcribe.cpp .gguf into the first ` +
+        `folder below, or set T3CODE_DICTATION_MODEL. Searched: ` +
         (input.directories.join(", ") || "nothing"),
     });
   }

apps/web/src/dictation/dictationLanguages.tsx

@@ -1,6 +1,6 @@
 import type { ReactNode } from "react";
 
-// Languages Parakeet v3 transcribes, each with the flag of its country. Drawn as SVG because
+// Languages the dictation models transcribe, each with the flag of its country. Drawn as SVG because
 // Windows renders flag emoji as two letters.
 const stripes = (direction: "row" | "column", colors: ReadonlyArray<string>) =>
   colors.map((color, index) => {