diff --git a/openbot/app/src/components/channels/chat-transcript.tsx b/openbot/app/src/components/channels/chat-transcript.tsx index 365495a8..114856dd 100644 --- a/openbot/app/src/components/channels/chat-transcript.tsx +++ b/openbot/app/src/components/channels/chat-transcript.tsx @@ -90,6 +90,7 @@ import { firstDayWelcome, firstNameOf, } from "./first-day-greeting"; +import { ReadAloudButton, ReadAloudContext, useReadAloud } from "./read-aloud"; import { ReplyHoldProvider } from "./reply-hold"; import { splitBubbles } from "./split-bubbles"; import { TeamBotConsentCard } from "./team-bot-consent"; @@ -105,6 +106,7 @@ import { VoiceChatCard } from "./voice-chat-card"; type ChatTranscriptProps = { busy?: boolean; + conversationId?: string; /** Comma-separated `/` command names, used to tell a skill chip from a leading slash. */ commandNames?: string; messages: ReadonlyArray>; @@ -677,7 +679,15 @@ export function firstDayParts(text: string): string[] { return parts.length > 0 ? parts : [text]; } -function FirstDayAnswer({ delay, text }: { delay: number; text: string }) { +function FirstDayAnswer({ + delay, + text, + action, +}: { + delay: number; + text: string; + action: ReactNode; +}) { return ( @@ -694,6 +704,7 @@ function FirstDayAnswer({ delay, text }: { delay: number; text: string }) { ))} + {action} ); @@ -851,6 +862,7 @@ function ResponsibilityFiring({ * It is also what keeps the entrance honest — no remount means no replay of the fade. */ const TranscriptMessage = memo(function TranscriptMessage({ + id, commandNames = "", delay, firstDayAnswer = false, @@ -859,6 +871,7 @@ const TranscriptMessage = memo(function TranscriptMessage({ seen = false, replay = false, }: { + id: string; commandNames?: string; delay: number; /** The Bot's answer to the first-day message, drawn one bubble per part. */ @@ -875,7 +888,13 @@ const TranscriptMessage = memo(function TranscriptMessage({ return

Trying again.

; } if (!isUser && firstDayAnswer) { - return ; + return ( + } + /> + ); } /* * Checked before anything else a person's message gets. A firing is not a person's message: the @@ -994,6 +1013,7 @@ const TranscriptMessage = memo(function TranscriptMessage({ )} {seen ? : null} + {!isUser && } @@ -1936,6 +1956,7 @@ function useSmoothSendScroll( } export function ChatTranscript({ + conversationId, busy = false, commandNames = "", messages, @@ -1947,6 +1968,10 @@ export function ChatTranscript({ contentClassName, folded = false, }: ChatTranscriptProps) { + const playback = useReadAloud( + conversationId, + Boolean(busy || restoring || folded), + ); const isDrawn = useDrawnTool(); /* * NOT MEMOISED, AND THAT IS DELIBERATE. `useMemo` keyed on `messages` looks obviously right and @@ -2079,7 +2104,7 @@ export function ChatTranscript({ } }, [hasItems, delays]); - return ( + const transcript = ( ); + return ( + + {transcript} + {playback.error && ( +

+ {playback.error} +

+ )} + + ); } diff --git a/openbot/app/src/components/channels/conversation-view.tsx b/openbot/app/src/components/channels/conversation-view.tsx index 2b8d3799..a31dd2b7 100644 --- a/openbot/app/src/components/channels/conversation-view.tsx +++ b/openbot/app/src/components/channels/conversation-view.tsx @@ -584,6 +584,7 @@ export function ConversationView({ * whole conversation to change nothing. */} void) | null = null; + +/** One owner per transcript. Old utterance callbacks never alter a newer session. */ +export function useReadAloud(scope: string | undefined, blocked: boolean) { + const current = useRef(null); + const [activeId, setActiveId] = useState(null); + const [error, setError] = useState(""); + const supported = + typeof window !== "undefined" && + typeof window.speechSynthesis?.speak === "function" && + typeof window.speechSynthesis.cancel === "function" && + typeof window.SpeechSynthesisUtterance === "function"; + const stop = useCallback((ownerId?: string) => { + if (ownerId !== undefined && current.current?.id !== ownerId) return; + const playback = current.current; + current.current = null; + if (stopCurrent === stop) stopCurrent = null; + if (playback) { + playback.utterance.onend = null; + playback.utterance.onerror = null; + window.speechSynthesis.cancel(); + } + setActiveId(null); + }, []); + useEffect(() => { + if (blocked) stop(); + }, [blocked, stop]); + // biome-ignore lint/correctness/useExhaustiveDependencies: changing conversation identity must cancel playback. + useEffect(() => { + setError(""); + const onHide = () => stop(); + window.addEventListener("pagehide", onHide); + return () => { + window.removeEventListener("pagehide", onHide); + stop(); + }; + }, [scope, stop]); + return { + activeId, + error, + supported, + blocked, + stop, + toggle(id: string, text: string) { + const same = current.current?.id === id; + stop(); + setError(""); + if (same || blocked || !supported) return; + const words = speechText(text); + if (!words) return; + stopCurrent?.(); + try { + const utterance = new window.SpeechSynthesisUtterance(words); + const playback = { id, utterance }; + current.current = playback; + stopCurrent = stop; + utterance.onend = () => { + if (current.current === playback) stop(); + }; + utterance.onerror = () => { + if (current.current !== playback) return; + stop(); + setError("Could not read this message aloud. Please try again."); + }; + setActiveId(id); + window.speechSynthesis.speak(utterance); + } catch { + stop(); + setError("Could not read this message aloud. Please try again."); + } + }, + }; +} + +export const ReadAloudContext = createContext | null>(null); + +export function ReadAloudButton({ id, text }: { id: string; text: string }) { + const playback = useContext(ReadAloudContext); + const active = playback?.activeId === id; + const stop = playback?.stop; + // biome-ignore lint/correctness/useExhaustiveDependencies: replacing source text must cancel its audio. + useEffect(() => { + // A removed or replaced message must release its audio, including on navigation. + return () => { + if (active) stop?.(id); + }; + }, [active, stop, text, id]); + const available = Boolean(playback?.supported && !playback.blocked); + const words = useMemo( + () => (available ? speechText(text) : ""), + [available, text], + ); + if (!playback || !words) return null; + const label = active ? "Stop reading" : "Read aloud"; + return ( + + + + ); +} diff --git a/openbot/app/src/lib/speech-text.ts b/openbot/app/src/lib/speech-text.ts new file mode 100644 index 00000000..ac8b85b2 --- /dev/null +++ b/openbot/app/src/lib/speech-text.ts @@ -0,0 +1,63 @@ +import { fromMarkdown } from "mdast-util-from-markdown"; +import { gfmFromMarkdown } from "mdast-util-gfm"; +import { gfm } from "micromark-extension-gfm"; + +type Node = { + type: string; + value?: string; + alt?: string | null; + url?: string; + children?: Node[]; +}; + +/** Speak visible prose, with pauses between blocks; omit markup, URLs and fenced code. */ +export function speechText(markdown: string): string { + const tree = fromMarkdown(markdown, { + extensions: [gfm()], + mdastExtensions: [gfmFromMarkdown()], + }); + function read(node: Node): string { + switch (node.type) { + case "html": + return /^$/i.test(node.value ?? "") ? "\n" : ""; + case "code": + case "definition": + case "footnoteDefinition": + case "footnoteReference": + return ""; + case "image": + case "imageReference": + return node.alt ?? ""; + case "link": { + const label = (node.children ?? []).map(read).join(""); + return label === node.url || label === node.url?.replace(/^mailto:/, "") + ? "" + : label; + } + case "text": + case "inlineCode": + return node.value ?? ""; + case "break": + case "thematicBreak": + return "\n"; + default: { + const blocks = [ + "root", + "blockquote", + "list", + "listItem", + "table", + "tableRow", + ]; + return (node.children ?? []) + .map(read) + .join(blocks.includes(node.type) ? "\n" : ""); + } + } + } + return read(tree) + .replace(/[^\S\n]+/g, " ") + .replace(/ *\n */g, "\n") + .replace(/\n{2,}/g, "\n") + .trim(); +} diff --git a/openbot/app/tests/read-aloud.test.tsx b/openbot/app/tests/read-aloud.test.tsx new file mode 100644 index 00000000..070b33b2 --- /dev/null +++ b/openbot/app/tests/read-aloud.test.tsx @@ -0,0 +1,309 @@ +import { afterAll, afterEach, beforeAll, expect, test } from "bun:test"; +import { CopilotKitContext } from "@copilotkit/react-core/v2/context"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, cleanup, fireEvent, render } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { StrictMode } from "react"; +import { ChatTranscript } from "@/components/channels/chat-transcript"; +import { + ReadAloudButton, + ReadAloudContext, + useReadAloud, +} from "@/components/channels/read-aloud"; +import { FIRST_DAY_REQUEST } from "@/components/shell/first-day"; +import { frameFirstDay } from "@/lib/channels/first-day"; +import { speechText } from "@/lib/speech-text"; + +let spoken: SpeechSynthesisUtterance[] = []; +let canceled = 0; +let fail = false; +class Utterance { + onend: (() => void) | null = null; + onerror: (() => void) | null = null; + constructor(public text: string) {} +} +beforeAll(() => GlobalRegistrator.register()); +afterEach(() => { + cleanup(); + spoken = []; + canceled = 0; + fail = false; +}); +afterAll(() => GlobalRegistrator.unregister()); +function install() { + Object.defineProperty(window, "SpeechSynthesisUtterance", { + configurable: true, + value: Utterance, + }); + Object.defineProperty(window, "speechSynthesis", { + configurable: true, + value: { + speak(utterance: SpeechSynthesisUtterance) { + if (fail) throw new Error("Unavailable"); + spoken.push(utterance); + }, + cancel() { + canceled++; + }, + }, + }); +} +function Harness({ + scope = "one", + blocked = false, + text = "**Hello**", + show = true, +}) { + const playback = useReadAloud(scope, blocked); + return ( + + {show && } + +

{playback.error}

+ + ); +} +test("speech keeps prose, labels, inline code and block boundaries, omitting code and destinations", () => { + expect( + speechText( + "# Hello\n\n**World** [label](https://example.com) `value` ![diagram](x)\n\n```js\nsecret()\n```\n\n- First\n- Second", + ), + ).toBe("Hello\nWorld label value diagram\nFirst\nSecond"); + expect( + speechText( + "[ref][id]\n\n[id]: https://example.com\n\n
\n\nWords\n\n
", + ), + ).toBe("ref\nWords"); + expect(speechText("| A | B |\n|---|---|\n| C | D |")).toBe("A\nB\nC\nD"); + expect(speechText(" \n```\ncode\n``` ")).toBe(""); +}); +test("click starts, switching cancels, stale callbacks cannot stop a new message, Stop cancels", () => { + install(); + const view = render(); + expect(spoken).toHaveLength(0); + fireEvent.click(view.getAllByRole("button", { name: "Read aloud" })[0]); + expect(spoken[0].text).toBe("Hello"); + const lateEnd = spoken[0].onend; + fireEvent.click(view.getByRole("button", { name: "Read aloud" })); + expect(spoken).toHaveLength(2); + expect(canceled).toBe(1); + act(() => lateEnd?.call(spoken[0], {} as SpeechSynthesisEvent)); + expect(view.getByRole("button", { name: "Stop reading" })).toBeTruthy(); + fireEvent.click(view.getByRole("button", { name: "Stop reading" })); + expect(canceled).toBe(2); +}); +test("completion and errors restore controls; thrown errors are announced", () => { + install(); + const view = render(); + fireEvent.click(view.getAllByRole("button")[0]); + act(() => spoken[0].onend?.({} as SpeechSynthesisEvent)); + expect(view.queryByRole("button", { name: "Stop reading" })).toBeNull(); + fireEvent.click(view.getAllByRole("button")[0]); + act(() => spoken[1].onerror?.({} as SpeechSynthesisErrorEvent)); + expect(view.getByRole("status").textContent).toContain("Could not read"); + fail = true; + fireEvent.click(view.getAllByRole("button")[0]); + expect(view.getByRole("status").textContent).toContain("Could not read"); +}); +for (const change of [ + "scope", + "blocked", + "text", + "show", + "unmount", + "pagehide", +] as const) { + test(`${change} stops playback`, () => { + install(); + const view = render(); + fireEvent.click(view.getAllByRole("button")[0]); + if (change === "unmount") view.unmount(); + else if (change === "pagehide") + act(() => window.dispatchEvent(new Event("pagehide"))); + else + view.rerender( + , + ); + expect(canceled).toBe(1); + }); +} +test("unsupported browsers, blocked playback and empty prose have no control", () => { + install(); + const view = render(); + expect(view.queryByRole("button")).toBeNull(); + view.rerender(); + expect(view.getAllByRole("button")).toHaveLength(1); + Object.defineProperty(window, "speechSynthesis", { + configurable: true, + value: undefined, + }); + view.rerender(); + expect(view.queryByRole("button")).toBeNull(); +}); + +test("transcript offers one icon per assistant message, including first-day answers, only when idle", () => { + install(); + const messages = [ + { id: "u", role: "user" as const, content: "Question" }, + { + id: "a", + role: "assistant" as const, + content: "One paragraph.\n\nAnother paragraph.", + }, + { id: "empty", role: "assistant" as const, content: "" }, + { + id: "tool", + role: "tool" as const, + toolCallId: "call", + content: "Tool output", + }, + ]; + const client = new QueryClient(); + const context = { + copilotkit: { + renderToolCalls: [], + subscribe: () => ({ unsubscribe() {} }), + }, + executingToolCallIds: new Set(), + } as never; + const draw = (busy: boolean, firstDay = false) => ( + + + + + + ); + const view = render(draw(false)); + const button = view.getByRole("button", { name: "Read aloud" }); + expect(button.textContent).toBe(""); + expect(button.getAttribute("title")).toBe("Read aloud"); + expect(button.tagName).toBe("BUTTON"); + expect(view.queryByRole("alert")).toBeNull(); + fail = true; + fireEvent.click(button); + expect(view.getByRole("alert").textContent).toContain("Could not read"); + expect(view.getByRole("alert").classList.contains("sr-only")).toBe(false); + fail = false; + view.rerender(draw(true)); + expect(view.queryByRole("button", { name: "Read aloud" })).toBeNull(); + view.rerender(draw(false, true)); + expect(view.getAllByRole("button", { name: "Read aloud" })).toHaveLength(1); +}); + +test("separate transcripts share playback ownership without canceling the new owner on unmount", () => { + install(); + const first = render(); + const second = render(); + fireEvent.click(first.container.querySelector("button")!); + const staleError = spoken[0].onerror; + fireEvent.click(second.container.querySelector("button")!); + expect(canceled).toBe(1); + expect( + first.container.querySelector('[aria-label="Stop reading"]'), + ).toBeNull(); + act(() => staleError?.call(spoken[0], {} as SpeechSynthesisErrorEvent)); + first.unmount(); + expect(canceled).toBe(1); + expect( + second.container.querySelector('[aria-label="Stop reading"]'), + ).toBeTruthy(); +}); + +test("URL-only links are omitted and HTML line breaks preserve word boundaries", () => { + expect( + speechText( + "See https://example.com/path and . [Guide](https://example.com)", + ), + ).toBe("See and . Guide"); + expect(speechText("Hello
world
again")).toBe("Hello\nworld\nagain"); +}); + +test("keyboard users can focus and start/stop speech with Enter and Space in Strict Mode", async () => { + install(); + const user = userEvent.setup({ document: window.document }); + const view = render( + + + , + ); + await user.tab(); + expect(document.activeElement).toBe(view.getAllByRole("button")[0]); + await user.keyboard("{Enter}"); + expect(spoken).toHaveLength(1); + expect(document.activeElement?.getAttribute("aria-label")).toBe( + "Stop reading", + ); + await user.keyboard(" "); + expect(canceled).toBe(1); + expect(view.queryByRole("button", { name: "Stop reading" })).toBeNull(); +}); + +test("rapid repeated requests do not autoplay or leave a stale error in a restarted message", () => { + install(); + const view = render(); + const button = view.getAllByRole("button")[0]; + fireEvent.click(button); + const staleError = spoken[0].onerror; + act(() => { + fireEvent.click(button); + fireEvent.click(button); + }); + expect(spoken).toHaveLength(2); + expect(canceled).toBe(1); + act(() => staleError?.call(spoken[0], {} as SpeechSynthesisErrorEvent)); + expect(view.getByRole("status").textContent).toBe(""); + expect(view.getByRole("button", { name: "Stop reading" })).toBeTruthy(); +}); + +test("natural speech preserves nested lists, escaping, multilingual prose and incomplete Markdown", () => { + expect( + speechText( + "## Plan\n\n> *Hello* and **welcome**.\n\n1. First\n - Nested\n2. Second\n\n\\*literal\\* café — مرحبا 世界", + ), + ).toBe( + "Plan\nHello and welcome.\nFirst\nNested\nSecond\n*literal* café — مرحبا 世界", + ); + expect(speechText("**Unfinished")).toBe("**Unfinished"); + expect(speechText("
hidden markup
")).toBe(""); + expect(speechText("before emphasis after")).toBe( + "before emphasis after", + ); + expect(speechText(" \n\n")).toBe(""); + expect(speechText("Long sentence. ".repeat(2000)).length).toBeGreaterThan( + 20000, + ); +}); + +test("conversation changes clear errors from the previous conversation", () => { + install(); + const view = render(); + fail = true; + fireEvent.click(view.getAllByRole("button")[0]); + expect(view.getByRole("status").textContent).toContain("Could not read"); + view.rerender(); + expect(view.getByRole("status").textContent).toBe(""); +});