diff --git a/package.json b/package.json index 58dac7a..ce95730 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "nixamp", - "version": "0.80.5", + "version": "0.80.6", "description": "It really whips the terminal's ass. A Winamp-shaped audio player for your terminal.", "license": "MIT", "type": "module", diff --git a/src/ai-label.ts b/src/ai-label.ts index ae7ba1c..2313f10 100644 --- a/src/ai-label.ts +++ b/src/ai-label.ts @@ -34,6 +34,8 @@ import { aiKey } from "./ai-keys.ts"; import { composerFromEnv, OutOfCredit, type Composer } from "./ai-radio.ts"; import { askJson, hasModel } from "./ai-text.ts"; import { formatLrc, parseLrc, type SyncedLine } from "./lyrics.ts"; +import { earAligner } from "./lyric-align.ts"; +import { readSession } from "./session.ts"; import { bandMarkdown, castLine, inventMembers, readMembers, writeMembers, type Member } from "./label-profiles.ts"; import type { LibraryIndex, Probed } from "./library-index.ts"; import type { StationRunner } from "./station-runner.ts"; @@ -3114,11 +3116,25 @@ export class AiLabel { * song's tags (seconds each), then old cover-and-waveform videos made again * as storyboards. */ - /** The aligner this server can use: the dependency, else ElevenLabs with its key, else none. */ + /** + * The aligner this server can use: the dependency, else ElevenLabs with its + * key and nixamp.com's ear when ElevenLabs refuses (a dead key answered 401 + * to every song, and the karaoke stopped), else the ear alone. + */ private aligner(): ((audio: string, text: string) => Promise) | null { if (this.deps.align !== undefined) return this.deps.align; + const ear = earAligner(() => readSession()); const key = aiKey("ELEVENLABS_API_KEY"); - return key ? elevenLabsAligner(key) : null; + if (!key) return ear; + const eleven = elevenLabsAligner(key); + return async (audio, text) => { + try { + return await eleven(audio, text); + } catch (error) { + this.log(`ElevenLabs could not time the words (${(error as Error).message}); asking nixamp.com's ear`); + return ear(audio, text); + } + }; } /** A song's words timed to its recording, written as 001. .lrc beside it. Once per song (a failure is not retried this run). */ diff --git a/src/lyric-align.ts b/src/lyric-align.ts new file mode 100644 index 0000000..396915f --- /dev/null +++ b/src/lyric-align.ts @@ -0,0 +1,161 @@ +/** + * A song's words timed to its recording without ElevenLabs: nixamp.com's ear + * (Whisper, POST /api/v1/speech/transcribe) hears the song a minute at a time + * and says when each phrase was sung; the known lyrics are then laid over + * what it heard, word by word, so every lyric word gets a time. + * + * Whisper mishears sung words (more so under guitars), so the heard text is + * only a guide: a lyric word matched to a heard one takes its time, and the + * words between two matches share the time between them. A song where too + * few words matched is refused rather than given a karaoke that drifts. + */ +import type { TimedWord } from "./ai-label.ts"; +import type { Session } from "./session.ts"; +import { RATE, type Segment } from "./speech.ts"; +import { askToHear, wavOfPcm, windowsOf, type Window } from "./transcribe.ts"; +import { isQuiet } from "./captions.ts"; + +/** A heard word, its time spread from its phrase by its length. */ +export interface HeardWord { word: string; start: number; end: number } + +/** What a word is for matching: lower case, letters and digits only. */ +export function wordKey(word: string): string { + return word.toLowerCase().normalize("NFKD").replace(/[̀-ͯ]/g, "").replace(/[^a-z0-9]/g, ""); +} + +/** Each phrase's words, its time shared out by how long each word is. */ +export function heardWords(segments: Segment[]): HeardWord[] { + const out: HeardWord[] = []; + for (const segment of segments) { + const words = segment.text.split(/\s+/).map((one) => ({ text: one, key: wordKey(one) })).filter((one) => one.key !== ""); + if (words.length === 0 || !(segment.end > segment.start)) continue; + const weight = words.reduce((sum, one) => sum + one.key.length + 1, 0); + let at = segment.start; + for (const one of words) { + const span = ((segment.end - segment.start) * (one.key.length + 1)) / weight; + out.push({ word: one.key, start: at, end: at + span }); + at += span; + } + } + return out; +} + +/** Whether a sung word and a heard one are the same word, allowing Whisper one slip in a longer word. */ +function same(a: string, b: string): boolean { + if (a === b) return true; + if (a.length < 4 || b.length < 4) return false; + if (a.slice(0, 4) === b.slice(0, 4) && Math.abs(a.length - b.length) <= 2) return true; + return Math.abs(a.length - b.length) <= 1 && slips(a, b) <= 1; +} + +function slips(a: string, b: string): number { + let row = Array.from({ length: b.length + 1 }, (_, j) => j); + for (let i = 1; i <= a.length; i++) { + const next = [i]; + for (let j = 1; j <= b.length; j++) next[j] = Math.min(row[j]! + 1, next[j - 1]! + 1, row[j - 1]! + (a[i - 1] === b[j - 1] ? 0 : 1)); + row = next; + } + return row[b.length]!; +} + +/** How long a word is sung, roughly, where nothing was heard to say. */ +const WORD_SECONDS = 0.35; + +/** + * One time per lyric word, in order: matched words (the longest common run + * of words, in order) keep the time they were heard at, the rest share out + * the time between their matched neighbours. Null when fewer than a fifth + * of the words (and at least three) matched. + */ +export function alignWords(lyrics: string[], heard: HeardWord[]): TimedWord[] | null { + const n = lyrics.length; + const m = heard.length; + if (n === 0 || m === 0) return null; + const keys = lyrics.map(wordKey); + // Longest common subsequence, as a table of lengths from the end. + const table = Array.from({ length: n + 1 }, () => new Uint16Array(m + 1)); + for (let i = n - 1; i >= 0; i--) { + for (let j = m - 1; j >= 0; j--) { + table[i]![j] = keys[i] !== "" && same(keys[i]!, heard[j]!.word) ? table[i + 1]![j + 1]! + 1 : Math.max(table[i + 1]![j]!, table[i]![j + 1]!); + } + } + const anchor: Array<HeardWord | null> = new Array(n).fill(null); + let matched = 0; + for (let i = 0, j = 0; i < n && j < m;) { + if (keys[i] !== "" && same(keys[i]!, heard[j]!.word) && table[i]![j] === table[i + 1]![j + 1]! + 1) { + anchor[i] = heard[j]!; + matched += 1; + i += 1; + j += 1; + } else if (table[i + 1]![j]! >= table[i]![j + 1]!) i += 1; + else j += 1; + } + if (matched < Math.max(3, Math.ceil(n * 0.2))) return null; + const starts: number[] = new Array(n).fill(0); + let previous = -1; + for (let i = 0; i <= n; i++) { + if (i < n && !anchor[i]) continue; + // The words between the last anchor and this one (or the ends of the song). + const from = previous; + const to = i; + for (let k = from + 1; k < to; k++) { + if (from >= 0 && to < n) { + const a = anchor[from]!.end; + const b = anchor[to]!.start; + starts[k] = a + ((b - a) * (k - from - 1)) / (to - from - 1); + } else if (from < 0 && to < n) { + starts[k] = Math.max(0, anchor[to]!.start - (to - k) * WORD_SECONDS); + } else { + starts[k] = anchor[from]!.end + (k - from - 1) * WORD_SECONDS; + } + } + if (i < n) starts[i] = anchor[i]!.start; + previous = i; + } + return lyrics.map((text, i) => { + const start = Math.round(starts[i]! * 100) / 100; + const own = anchor[i] ? anchor[i]!.end : start + WORD_SECONDS; + const next = i + 1 < n ? starts[i + 1]! : own; + const end = Math.round(Math.max(start + 0.05, Math.min(own, next > start ? next : own)) * 100) / 100; + return { text, start, end }; + }); +} + +export interface EarDeps { + fetcher?: typeof fetch; + windows?: (source: string) => AsyncIterable<Window>; + sleep?: (ms: number) => Promise<void>; +} + +/** Every sung phrase of a recording with its time, heard by nixamp.com's ear a window at a time. */ +export async function hearSong(session: Pick<Session, "site" | "token">, audio: string, language: string, deps: EarDeps = {}): Promise<Segment[]> { + const sleep = deps.sleep ?? ((ms: number) => new Promise<void>((done) => setTimeout(done, ms))); + const out: Segment[] = []; + for await (const window of (deps.windows ?? ((path: string) => windowsOf(path)))(audio)) { + if (isQuiet(window.pcm)) continue; + const wav = wavOfPcm(window.pcm); + let answer = await askToHear(session, { wav, timestamps: true, ...(language ? { language } : {}) }, deps.fetcher); + // The ear allows so many seconds of sound a minute: a 429 is a wait. + for (let attempt = 0; !answer.ok && answer.status === 429 && attempt < 30; attempt++) { + await sleep(10_000); + answer = await askToHear(session, { wav, timestamps: true, ...(language ? { language } : {}) }, deps.fetcher); + } + if (!answer.ok) throw new Error(`the ear answered ${answer.status || "nothing"}: ${answer.error}`); + const length = window.pcm.length / (RATE * 2); + const pieces = answer.heard.segments?.length ? answer.heard.segments : answer.heard.text ? [{ start: 0, end: length, text: answer.heard.text }] : []; + for (const piece of pieces) out.push({ start: window.offset + piece.start, end: window.offset + Math.min(piece.end, length), text: piece.text }); + } + return out; +} + +/** An aligner like ElevenLabs': the song and its words in, one time per word out, heard by nixamp.com. */ +export function earAligner(session: () => Pick<Session, "site" | "token"> | null, language = "en", deps: EarDeps = {}): (audio: string, text: string) => Promise<TimedWord[]> { + return async (audio, text) => { + const signedIn = session(); + if (!signedIn?.token) throw new Error("this server is not signed in to nixamp.com, so its ear cannot time the words; `nixamp login`"); + const words = text.split(/\s+/).filter(Boolean); + const timed = alignWords(words, heardWords(await hearSong(signedIn, audio, language, deps))); + if (!timed) throw new Error("the ear heard too few of the words to time them"); + return timed; + }; +} diff --git a/src/transcribe.ts b/src/transcribe.ts index d26eeff..26f9e79 100644 --- a/src/transcribe.ts +++ b/src/transcribe.ts @@ -195,7 +195,7 @@ export async function askToHear(session: Pick<Session, "site" | "token">, ask: A } /** A WAV around 16-bit mono PCM at RATE, for one window. */ -function wavOfPcm(pcm: Buffer): Uint8Array { +export function wavOfPcm(pcm: Buffer): Uint8Array { const header = Buffer.alloc(44); header.write("RIFF", 0, "ascii"); header.writeUInt32LE(36 + pcm.length, 4); diff --git a/test/lyric-align.test.ts b/test/lyric-align.test.ts new file mode 100644 index 0000000..efb2dae --- /dev/null +++ b/test/lyric-align.test.ts @@ -0,0 +1,59 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { alignWords, earAligner, heardWords, wordKey } from "../src/lyric-align.ts"; + +test("a heard phrase shares its time among its words by length", () => { + const words = heardWords([{ start: 10, end: 14, text: "Cut the clock!" }]); + assert.deepEqual(words.map((one) => one.word), ["cut", "the", "clock"]); + assert.equal(words[0]!.start, 10); + assert.ok(Math.abs(words[2]!.end - 14) < 1e-9); + assert.ok(words[2]!.end - words[2]!.start > words[0]!.end - words[0]!.start, "a longer word is sung longer"); + assert.equal(wordKey("Couldn’t,"), "couldnt"); +}); + +test("every lyric word gets a time: matched ones as heard, the rest in between", () => { + const lyrics = "They sold the morning by the minute sold the evening by the week".split(" "); + // Whisper misheard "Sold" as "So" and dropped "by the week". + const heard = heardWords([ + { start: 45, end: 47, text: "They sold the morning by the minute" }, + { start: 47, end: 48, text: "So the evening" }, + ]); + const timed = alignWords(lyrics, heard)!; + assert.equal(timed.length, lyrics.length); + assert.deepEqual(timed.map((one) => one.text), lyrics); + assert.equal(timed[0]!.start, 45); + for (let i = 1; i < timed.length; i++) assert.ok(timed[i]!.start >= timed[i - 1]!.start, `in order at ${i}`); + assert.ok(timed[8]!.start >= 47 && timed[8]!.start < 48, "evening as heard"); + assert.ok(timed[11]!.start > timed[8]!.start, "the unheard tail carries on after the last match"); +}); + +test("a song where too little was heard is refused, not given a karaoke that drifts", () => { + const lyrics = "one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen".split(" "); + assert.equal(alignWords(lyrics, heardWords([{ start: 0, end: 5, text: "one banana" }])), null); + assert.equal(alignWords(lyrics, []), null); +}); + +test("the ear aligner hears a song a minute at a time and waits out the ear's 429", async () => { + const pcm = Buffer.alloc(16_000 * 2 * 2); + for (let i = 0; i < pcm.length; i += 2) pcm.writeInt16LE(Math.round(Math.sin(i / 20) * 12000), i); + let asks = 0; + const fetcher = (async () => { + asks += 1; + if (asks === 1) return new Response(JSON.stringify({ error: "busy" }), { status: 429 }); + const second = asks === 2 ? 0 : 60; + return new Response(JSON.stringify({ text: "x", seconds: 2, segments: second === 0 + ? [{ start: 0, end: 2, text: "Your time was never theirs" }] + : [{ start: 0.5, end: 1.5, text: "to own" }] }), { status: 200 }); + }) as unknown as typeof fetch; + async function* windows() { + yield { offset: 0, pcm }; + yield { offset: 60, pcm }; + } + const align = earAligner(() => ({ site: "https://nixamp.com", token: "t" }), "en", { fetcher, windows, sleep: async () => undefined }); + const timed = await align("song.mp3", "Your time was never\ntheirs to own"); + assert.equal(asks, 3); + assert.deepEqual(timed.map((one) => one.text), ["Your", "time", "was", "never", "theirs", "to", "own"]); + assert.equal(timed[0]!.start, 0); + assert.ok(timed[5]!.start >= 60.5, "the second window's words are offset by where it starts"); + await assert.rejects(earAligner(() => null)("song.mp3", "words"), /not signed in/); +}); diff --git a/test/paywall.test.ts b/test/paywall.test.ts index 10981c1..f7448b9 100644 --- a/test/paywall.test.ts +++ b/test/paywall.test.ts @@ -161,12 +161,8 @@ test("the crawl door is off without a key and a payee", () => { test("the crawl door charges GPTBot with a PAYMENT-REQUIRED header", async () => { const gate = createCrawlGate(CRAWL_ENV)!; - // x402-gateway gives a training crawler 100 free requests a minute per address to evaluate the site; past them it pays. - const ask = () => gate.handle(new Request("http://localhost/music", { headers: { "user-agent": GPTBOT, accept: "application/json", "x-real-ip": "203.0.113.7" } })); - let answer = await ask(); - let free = 0; - for (; free < 200 && !answer; free++) answer = await ask(); - assert.ok(free >= 1, "the first look is free"); + // Since x402-gateway 0.18.1 a training crawler pays from its first request (#432). + const answer = await gate.handle(new Request("http://localhost/music", { headers: { "user-agent": GPTBOT, accept: "application/json", "x-real-ip": "203.0.113.7" } })); assert.equal(answer?.status, 402); assert.ok(answer?.headers.get("payment-required")); });