Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "nixamp",
"version": "0.80.5",
"version": "0.80.6",
"description": "It really whips the terminal's ass. A Winamp-shaped audio player for your terminal.",
"license": "MIT",
"type": "module",
Expand Down
20 changes: 18 additions & 2 deletions src/ai-label.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ import { aiKey } from "./ai-keys.ts";
import { composerFromEnv, OutOfCredit, type Composer } from "./ai-radio.ts";
import { askJson, hasModel } from "./ai-text.ts";
import { formatLrc, parseLrc, type SyncedLine } from "./lyrics.ts";
import { earAligner } from "./lyric-align.ts";
import { readSession } from "./session.ts";
import { bandMarkdown, castLine, inventMembers, readMembers, writeMembers, type Member } from "./label-profiles.ts";
import type { LibraryIndex, Probed } from "./library-index.ts";
import type { StationRunner } from "./station-runner.ts";
Expand Down Expand Up @@ -3114,11 +3116,25 @@ export class AiLabel {
* song's tags (seconds each), then old cover-and-waveform videos made again
* as storyboards.
*/
/** The aligner this server can use: the dependency, else ElevenLabs with its key, else none. */
/**
* The aligner this server can use: the dependency, else ElevenLabs with its
* key and nixamp.com's ear when ElevenLabs refuses (a dead key answered 401
* to every song, and the karaoke stopped), else the ear alone.
*/
private aligner(): ((audio: string, text: string) => Promise<TimedWord[]>) | null {
if (this.deps.align !== undefined) return this.deps.align;
const ear = earAligner(() => readSession());
const key = aiKey("ELEVENLABS_API_KEY");
return key ? elevenLabsAligner(key) : null;
if (!key) return ear;
const eleven = elevenLabsAligner(key);
return async (audio, text) => {
try {
return await eleven(audio, text);
} catch (error) {
this.log(`ElevenLabs could not time the words (${(error as Error).message}); asking nixamp.com's ear`);
return ear(audio, text);
}
};
}

/** A song's words timed to its recording, written as 001. <title>.lrc beside it. Once per song (a failure is not retried this run). */
Expand Down
161 changes: 161 additions & 0 deletions src/lyric-align.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
/**
* A song's words timed to its recording without ElevenLabs: nixamp.com's ear
* (Whisper, POST /api/v1/speech/transcribe) hears the song a minute at a time
* and says when each phrase was sung; the known lyrics are then laid over
* what it heard, word by word, so every lyric word gets a time.
*
* Whisper mishears sung words (more so under guitars), so the heard text is
* only a guide: a lyric word matched to a heard one takes its time, and the
* words between two matches share the time between them. A song where too
* few words matched is refused rather than given a karaoke that drifts.
*/
import type { TimedWord } from "./ai-label.ts";
import type { Session } from "./session.ts";
import { RATE, type Segment } from "./speech.ts";
import { askToHear, wavOfPcm, windowsOf, type Window } from "./transcribe.ts";
import { isQuiet } from "./captions.ts";

/** A heard word, its time spread from its phrase by its length. */
export interface HeardWord { word: string; start: number; end: number }

/** What a word is for matching: lower case, letters and digits only. */
export function wordKey(word: string): string {
return word.toLowerCase().normalize("NFKD").replace(/[̀-ͯ]/g, "").replace(/[^a-z0-9]/g, "");
}

/** Each phrase's words, its time shared out by how long each word is. */
export function heardWords(segments: Segment[]): HeardWord[] {
const out: HeardWord[] = [];
for (const segment of segments) {
const words = segment.text.split(/\s+/).map((one) => ({ text: one, key: wordKey(one) })).filter((one) => one.key !== "");
if (words.length === 0 || !(segment.end > segment.start)) continue;
const weight = words.reduce((sum, one) => sum + one.key.length + 1, 0);
let at = segment.start;
for (const one of words) {
const span = ((segment.end - segment.start) * (one.key.length + 1)) / weight;
out.push({ word: one.key, start: at, end: at + span });
at += span;
}
}
return out;
}

/** Whether a sung word and a heard one are the same word, allowing Whisper one slip in a longer word. */
function same(a: string, b: string): boolean {
if (a === b) return true;
if (a.length < 4 || b.length < 4) return false;
if (a.slice(0, 4) === b.slice(0, 4) && Math.abs(a.length - b.length) <= 2) return true;
return Math.abs(a.length - b.length) <= 1 && slips(a, b) <= 1;
}

function slips(a: string, b: string): number {
let row = Array.from({ length: b.length + 1 }, (_, j) => j);
for (let i = 1; i <= a.length; i++) {
const next = [i];
for (let j = 1; j <= b.length; j++) next[j] = Math.min(row[j]! + 1, next[j - 1]! + 1, row[j - 1]! + (a[i - 1] === b[j - 1] ? 0 : 1));
row = next;
}
return row[b.length]!;
}

/** How long a word is sung, roughly, where nothing was heard to say. */
const WORD_SECONDS = 0.35;

/**
* One time per lyric word, in order: matched words (the longest common run
* of words, in order) keep the time they were heard at, the rest share out
* the time between their matched neighbours. Null when fewer than a fifth
* of the words (and at least three) matched.
*/
export function alignWords(lyrics: string[], heard: HeardWord[]): TimedWord[] | null {
const n = lyrics.length;
const m = heard.length;
if (n === 0 || m === 0) return null;
const keys = lyrics.map(wordKey);
// Longest common subsequence, as a table of lengths from the end.
const table = Array.from({ length: n + 1 }, () => new Uint16Array(m + 1));
for (let i = n - 1; i >= 0; i--) {
for (let j = m - 1; j >= 0; j--) {
table[i]![j] = keys[i] !== "" && same(keys[i]!, heard[j]!.word) ? table[i + 1]![j + 1]! + 1 : Math.max(table[i + 1]![j]!, table[i]![j + 1]!);
}
}
const anchor: Array<HeardWord | null> = new Array(n).fill(null);
let matched = 0;
for (let i = 0, j = 0; i < n && j < m;) {
if (keys[i] !== "" && same(keys[i]!, heard[j]!.word) && table[i]![j] === table[i + 1]![j + 1]! + 1) {
anchor[i] = heard[j]!;
matched += 1;
i += 1;
j += 1;
} else if (table[i + 1]![j]! >= table[i]![j + 1]!) i += 1;
else j += 1;
}
if (matched < Math.max(3, Math.ceil(n * 0.2))) return null;
const starts: number[] = new Array(n).fill(0);
let previous = -1;
for (let i = 0; i <= n; i++) {
if (i < n && !anchor[i]) continue;
// The words between the last anchor and this one (or the ends of the song).
const from = previous;
const to = i;
for (let k = from + 1; k < to; k++) {
if (from >= 0 && to < n) {
const a = anchor[from]!.end;
const b = anchor[to]!.start;
starts[k] = a + ((b - a) * (k - from - 1)) / (to - from - 1);
} else if (from < 0 && to < n) {
starts[k] = Math.max(0, anchor[to]!.start - (to - k) * WORD_SECONDS);
} else {
starts[k] = anchor[from]!.end + (k - from - 1) * WORD_SECONDS;
}
}
if (i < n) starts[i] = anchor[i]!.start;
previous = i;
}
return lyrics.map((text, i) => {
const start = Math.round(starts[i]! * 100) / 100;
const own = anchor[i] ? anchor[i]!.end : start + WORD_SECONDS;
const next = i + 1 < n ? starts[i + 1]! : own;
const end = Math.round(Math.max(start + 0.05, Math.min(own, next > start ? next : own)) * 100) / 100;
return { text, start, end };
});
}

export interface EarDeps {
fetcher?: typeof fetch;
windows?: (source: string) => AsyncIterable<Window>;
sleep?: (ms: number) => Promise<void>;
}

/** Every sung phrase of a recording with its time, heard by nixamp.com's ear a window at a time. */
export async function hearSong(session: Pick<Session, "site" | "token">, audio: string, language: string, deps: EarDeps = {}): Promise<Segment[]> {
const sleep = deps.sleep ?? ((ms: number) => new Promise<void>((done) => setTimeout(done, ms)));
const out: Segment[] = [];
for await (const window of (deps.windows ?? ((path: string) => windowsOf(path)))(audio)) {
if (isQuiet(window.pcm)) continue;
const wav = wavOfPcm(window.pcm);
let answer = await askToHear(session, { wav, timestamps: true, ...(language ? { language } : {}) }, deps.fetcher);
// The ear allows so many seconds of sound a minute: a 429 is a wait.
for (let attempt = 0; !answer.ok && answer.status === 429 && attempt < 30; attempt++) {
await sleep(10_000);
answer = await askToHear(session, { wav, timestamps: true, ...(language ? { language } : {}) }, deps.fetcher);
}
if (!answer.ok) throw new Error(`the ear answered ${answer.status || "nothing"}: ${answer.error}`);
const length = window.pcm.length / (RATE * 2);
const pieces = answer.heard.segments?.length ? answer.heard.segments : answer.heard.text ? [{ start: 0, end: length, text: answer.heard.text }] : [];
for (const piece of pieces) out.push({ start: window.offset + piece.start, end: window.offset + Math.min(piece.end, length), text: piece.text });
}
return out;
}

/** An aligner like ElevenLabs': the song and its words in, one time per word out, heard by nixamp.com. */
export function earAligner(session: () => Pick<Session, "site" | "token"> | null, language = "en", deps: EarDeps = {}): (audio: string, text: string) => Promise<TimedWord[]> {
return async (audio, text) => {
const signedIn = session();
if (!signedIn?.token) throw new Error("this server is not signed in to nixamp.com, so its ear cannot time the words; `nixamp login`");
const words = text.split(/\s+/).filter(Boolean);
const timed = alignWords(words, heardWords(await hearSong(signedIn, audio, language, deps)));
if (!timed) throw new Error("the ear heard too few of the words to time them");
return timed;
};
}
2 changes: 1 addition & 1 deletion src/transcribe.ts
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ export async function askToHear(session: Pick<Session, "site" | "token">, ask: A
}

/** A WAV around 16-bit mono PCM at RATE, for one window. */
function wavOfPcm(pcm: Buffer): Uint8Array {
export function wavOfPcm(pcm: Buffer): Uint8Array {
const header = Buffer.alloc(44);
header.write("RIFF", 0, "ascii");
header.writeUInt32LE(36 + pcm.length, 4);
Expand Down
59 changes: 59 additions & 0 deletions test/lyric-align.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { alignWords, earAligner, heardWords, wordKey } from "../src/lyric-align.ts";

test("a heard phrase shares its time among its words by length", () => {
const words = heardWords([{ start: 10, end: 14, text: "Cut the clock!" }]);
assert.deepEqual(words.map((one) => one.word), ["cut", "the", "clock"]);
assert.equal(words[0]!.start, 10);
assert.ok(Math.abs(words[2]!.end - 14) < 1e-9);
assert.ok(words[2]!.end - words[2]!.start > words[0]!.end - words[0]!.start, "a longer word is sung longer");
assert.equal(wordKey("Couldn’t,"), "couldnt");
});

test("every lyric word gets a time: matched ones as heard, the rest in between", () => {
const lyrics = "They sold the morning by the minute sold the evening by the week".split(" ");
// Whisper misheard "Sold" as "So" and dropped "by the week".
const heard = heardWords([
{ start: 45, end: 47, text: "They sold the morning by the minute" },
{ start: 47, end: 48, text: "So the evening" },
]);
const timed = alignWords(lyrics, heard)!;
assert.equal(timed.length, lyrics.length);
assert.deepEqual(timed.map((one) => one.text), lyrics);
assert.equal(timed[0]!.start, 45);
for (let i = 1; i < timed.length; i++) assert.ok(timed[i]!.start >= timed[i - 1]!.start, `in order at ${i}`);
assert.ok(timed[8]!.start >= 47 && timed[8]!.start < 48, "evening as heard");
assert.ok(timed[11]!.start > timed[8]!.start, "the unheard tail carries on after the last match");
});

test("a song where too little was heard is refused, not given a karaoke that drifts", () => {
const lyrics = "one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen".split(" ");
assert.equal(alignWords(lyrics, heardWords([{ start: 0, end: 5, text: "one banana" }])), null);
assert.equal(alignWords(lyrics, []), null);
});

test("the ear aligner hears a song a minute at a time and waits out the ear's 429", async () => {
const pcm = Buffer.alloc(16_000 * 2 * 2);
for (let i = 0; i < pcm.length; i += 2) pcm.writeInt16LE(Math.round(Math.sin(i / 20) * 12000), i);
let asks = 0;
const fetcher = (async () => {
asks += 1;
if (asks === 1) return new Response(JSON.stringify({ error: "busy" }), { status: 429 });
const second = asks === 2 ? 0 : 60;
return new Response(JSON.stringify({ text: "x", seconds: 2, segments: second === 0
? [{ start: 0, end: 2, text: "Your time was never theirs" }]
: [{ start: 0.5, end: 1.5, text: "to own" }] }), { status: 200 });
}) as unknown as typeof fetch;
async function* windows() {
yield { offset: 0, pcm };
yield { offset: 60, pcm };
}
const align = earAligner(() => ({ site: "https://nixamp.com", token: "t" }), "en", { fetcher, windows, sleep: async () => undefined });
const timed = await align("song.mp3", "Your time was never\ntheirs to own");
assert.equal(asks, 3);
assert.deepEqual(timed.map((one) => one.text), ["Your", "time", "was", "never", "theirs", "to", "own"]);
assert.equal(timed[0]!.start, 0);
assert.ok(timed[5]!.start >= 60.5, "the second window's words are offset by where it starts");
await assert.rejects(earAligner(() => null)("song.mp3", "words"), /not signed in/);
});
8 changes: 2 additions & 6 deletions test/paywall.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -161,12 +161,8 @@ test("the crawl door is off without a key and a payee", () => {

test("the crawl door charges GPTBot with a PAYMENT-REQUIRED header", async () => {
const gate = createCrawlGate(CRAWL_ENV)!;
// x402-gateway gives a training crawler 100 free requests a minute per address to evaluate the site; past them it pays.
const ask = () => gate.handle(new Request("http://localhost/music", { headers: { "user-agent": GPTBOT, accept: "application/json", "x-real-ip": "203.0.113.7" } }));
let answer = await ask();
let free = 0;
for (; free < 200 && !answer; free++) answer = await ask();
assert.ok(free >= 1, "the first look is free");
// Since x402-gateway 0.18.1 a training crawler pays from its first request (#432).
const answer = await gate.handle(new Request("http://localhost/music", { headers: { "user-agent": GPTBOT, accept: "application/json", "x-real-ip": "203.0.113.7" } }));
assert.equal(answer?.status, 402);
assert.ok(answer?.headers.get("payment-required"));
});
Expand Down
Loading