feat(chat): add read-aloud support for assistant messages

openbot/app/src/components/channels/chat-transcript.tsx

@@ -90,6 +90,7 @@ import {
   firstDayWelcome,
   firstNameOf,
 } from "./first-day-greeting";
+import { ReadAloudButton, ReadAloudContext, useReadAloud } from "./read-aloud";
 import { ReplyHoldProvider } from "./reply-hold";
 import { splitBubbles } from "./split-bubbles";
 import { TeamBotConsentCard } from "./team-bot-consent";
@@ -105,6 +106,7 @@ import { VoiceChatCard } from "./voice-chat-card";
 
 type ChatTranscriptProps = {
   busy?: boolean;
+  conversationId?: string;
   /** Comma-separated `/` command names, used to tell a skill chip from a leading slash. */
   commandNames?: string;
   messages: ReadonlyArray<Readonly<Message>>;
@@ -677,7 +679,15 @@ export function firstDayParts(text: string): string[] {
   return parts.length > 0 ? parts : [text];
 }
 
-function FirstDayAnswer({ delay, text }: { delay: number; text: string }) {
+function FirstDayAnswer({
+  delay,
+  text,
+  action,
+}: {
+  delay: number;
+  text: string;
+  action: ReactNode;
+}) {
   return (
     <MessageRow align="start">
       <MessageContent>
@@ -694,6 +704,7 @@ function FirstDayAnswer({ delay, text }: { delay: number; text: string }) {
             </Bubble>
           </Arriving>
         ))}
+        {action}
       </MessageContent>
     </MessageRow>
   );
@@ -851,6 +862,7 @@ function ResponsibilityFiring({
  * It is also what keeps the entrance honest — no remount means no replay of the fade.
  */
 const TranscriptMessage = memo(function TranscriptMessage({
+  id,
   commandNames = "",
   delay,
   firstDayAnswer = false,
@@ -859,6 +871,7 @@ const TranscriptMessage = memo(function TranscriptMessage({
   seen = false,
   replay = false,
 }: {
+  id: string;
   commandNames?: string;
   delay: number;
   /** The Bot's answer to the first-day message, drawn one bubble per part. */
@@ -875,7 +888,13 @@ const TranscriptMessage = memo(function TranscriptMessage({
     return <p className="type-label text-duet-ink-3">Trying again.</p>;
   }
   if (!isUser && firstDayAnswer) {
-    return <FirstDayAnswer delay={delay} text={text} />;
+    return (
+      <FirstDayAnswer
+        delay={delay}
+        text={text}
+        action={<ReadAloudButton id={id} text={text} />}
+      />
+    );
   }
   /*
    * Checked before anything else a person's message gets. A firing is not a person's message: the
@@ -994,6 +1013,7 @@ const TranscriptMessage = memo(function TranscriptMessage({
             </div>
           )}
           {seen ? <Seen /> : null}
+          {!isUser && <ReadAloudButton id={id} text={text} />}
         </Arriving>
       </MessageContent>
     </MessageRow>
@@ -1936,6 +1956,7 @@ function useSmoothSendScroll(
 }
 
 export function ChatTranscript({
+  conversationId,
   busy = false,
   commandNames = "",
   messages,
@@ -1947,6 +1968,10 @@ export function ChatTranscript({
   contentClassName,
   folded = false,
 }: ChatTranscriptProps) {
+  const playback = useReadAloud(
+    conversationId,
+    Boolean(busy || restoring || folded),
+  );
   const isDrawn = useDrawnTool();
   /*
    * NOT MEMOISED, AND THAT IS DELIBERATE. `useMemo` keyed on `messages` looks obviously right and
@@ -2079,7 +2104,7 @@ export function ChatTranscript({
     }
   }, [hasItems, delays]);
 
-  return (
+  const transcript = (
     <MessageScrollerProvider
       autoScroll={!folded && !briefArriving}
       scrollPreviousItemPeek={48}
@@ -2224,6 +2249,7 @@ export function ChatTranscript({
                   scrollAnchor={anchorRows.has(item.id)}
                 >
                   <TranscriptMessage
+                    id={item.id}
                     commandNames={commandNames}
                     delay={delays.delayFor(item.id, index, items.length)}
                     firstDayAnswer={firstDay?.answer.has(item.id) === true}
@@ -2264,4 +2290,14 @@ export function ChatTranscript({
       </MessageScroller>
     </MessageScrollerProvider>
   );
+  return (
+    <ReadAloudContext.Provider value={playback}>
+      {transcript}
+      {playback.error && (
+        <p role="alert" className="text-sm text-destructive">
+          {playback.error}
+        </p>
+      )}
+    </ReadAloudContext.Provider>
+  );
 }

openbot/app/src/components/channels/conversation-view.tsx

@@ -584,6 +584,7 @@ export function ConversationView({
            * whole conversation to change nothing.
            */}
           <ChatTranscript
+            conversationId={channelId}
             busy={busy}
             {...(inShell
               ? {

openbot/app/src/components/channels/read-aloud.tsx

@@ -0,0 +1,135 @@
+import { IconPlayerStop, IconVolume } from "@tabler/icons-react";
+import {
+  createContext,
+  useCallback,
+  useContext,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+} from "react";
+import { Button } from "@/components/ui/button";
+import { MessageFooter } from "@/components/ui/message";
+import { speechText } from "@/lib/speech-text";
+
+type Playback = {
+  id: string;
+  utterance: SpeechSynthesisUtterance;
+};
+
+// SpeechSynthesis has one queue per window, even when several transcripts are mounted.
+let stopCurrent: (() => void) | null = null;
+
+/** One owner per transcript. Old utterance callbacks never alter a newer session. */
+export function useReadAloud(scope: string | undefined, blocked: boolean) {
+  const current = useRef<Playback | null>(null);
+  const [activeId, setActiveId] = useState<string | null>(null);
+  const [error, setError] = useState("");
+  const supported =
+    typeof window !== "undefined" &&
+    typeof window.speechSynthesis?.speak === "function" &&
+    typeof window.speechSynthesis.cancel === "function" &&
+    typeof window.SpeechSynthesisUtterance === "function";
+  const stop = useCallback((ownerId?: string) => {
+    if (ownerId !== undefined && current.current?.id !== ownerId) return;
+    const playback = current.current;
+    current.current = null;
+    if (stopCurrent === stop) stopCurrent = null;
+    if (playback) {
+      playback.utterance.onend = null;
+      playback.utterance.onerror = null;
+      window.speechSynthesis.cancel();
+    }
+    setActiveId(null);
+  }, []);
+  useEffect(() => {
+    if (blocked) stop();
+  }, [blocked, stop]);
+  // biome-ignore lint/correctness/useExhaustiveDependencies: changing conversation identity must cancel playback.
+  useEffect(() => {
+    setError("");
+    const onHide = () => stop();
+    window.addEventListener("pagehide", onHide);
+    return () => {
+      window.removeEventListener("pagehide", onHide);
+      stop();
+    };
+  }, [scope, stop]);
+  return {
+    activeId,
+    error,
+    supported,
+    blocked,
+    stop,
+    toggle(id: string, text: string) {
+      const same = current.current?.id === id;
+      stop();
+      setError("");
+      if (same || blocked || !supported) return;
+      const words = speechText(text);
+      if (!words) return;
+      stopCurrent?.();
+      try {
+        const utterance = new window.SpeechSynthesisUtterance(words);
+        const playback = { id, utterance };
+        current.current = playback;
+        stopCurrent = stop;
+        utterance.onend = () => {
+          if (current.current === playback) stop();
+        };
+        utterance.onerror = () => {
+          if (current.current !== playback) return;
+          stop();
+          setError("Could not read this message aloud. Please try again.");
+        };
+        setActiveId(id);
+        window.speechSynthesis.speak(utterance);
+      } catch {
+        stop();
+        setError("Could not read this message aloud. Please try again.");
+      }
+    },
+  };
+}
+
+export const ReadAloudContext = createContext<ReturnType<
+  typeof useReadAloud
+> | null>(null);
+
+export function ReadAloudButton({ id, text }: { id: string; text: string }) {
+  const playback = useContext(ReadAloudContext);
+  const active = playback?.activeId === id;
+  const stop = playback?.stop;
+  // biome-ignore lint/correctness/useExhaustiveDependencies: replacing source text must cancel its audio.
+  useEffect(() => {
+    // A removed or replaced message must release its audio, including on navigation.
+    return () => {
+      if (active) stop?.(id);
+    };
+  }, [active, stop, text, id]);
+  const available = Boolean(playback?.supported && !playback.blocked);
+  const words = useMemo(
+    () => (available ? speechText(text) : ""),
+    [available, text],
+  );
+  if (!playback || !words) return null;
+  const label = active ? "Stop reading" : "Read aloud";
+  return (
+    <MessageFooter>
+      <Button
+        aria-label={label}
+        title={label}
+        variant="ghost"
+        size="icon"
+        className="text-duet-ink-3 hover:text-duet-ink"
+        onClick={() => playback.toggle(id, text)}
+      >
+        {active ? (
+          <IconPlayerStop aria-hidden="true" />
+        ) : (
+          <IconVolume aria-hidden="true" />
+        )}
+      </Button>
+    </MessageFooter>
+  );
+}

openbot/app/src/lib/speech-text.ts

@@ -0,0 +1,63 @@
+import { fromMarkdown } from "mdast-util-from-markdown";
+import { gfmFromMarkdown } from "mdast-util-gfm";
+import { gfm } from "micromark-extension-gfm";
+
+type Node = {
+  type: string;
+  value?: string;
+  alt?: string | null;
+  url?: string;
+  children?: Node[];
+};
+
+/** Speak visible prose, with pauses between blocks; omit markup, URLs and fenced code. */
+export function speechText(markdown: string): string {
+  const tree = fromMarkdown(markdown, {
+    extensions: [gfm()],
+    mdastExtensions: [gfmFromMarkdown()],
+  });
+  function read(node: Node): string {
+    switch (node.type) {
+      case "html":
+        return /^<br\s*\/?\s*>$/i.test(node.value ?? "") ? "\n" : "";
+      case "code":
+      case "definition":
+      case "footnoteDefinition":
+      case "footnoteReference":
+        return "";
+      case "image":
+      case "imageReference":
+        return node.alt ?? "";
+      case "link": {
+        const label = (node.children ?? []).map(read).join("");
+        return label === node.url || label === node.url?.replace(/^mailto:/, "")
+          ? ""
+          : label;
+      }
+      case "text":
+      case "inlineCode":
+        return node.value ?? "";
+      case "break":
+      case "thematicBreak":
+        return "\n";
+      default: {
+        const blocks = [
+          "root",
+          "blockquote",
+          "list",
+          "listItem",
+          "table",
+          "tableRow",
+        ];
+        return (node.children ?? [])
+          .map(read)
+          .join(blocks.includes(node.type) ? "\n" : "");
+      }
+    }
+  }
+  return read(tree)
+    .replace(/[^\S\n]+/g, " ")
+    .replace(/ *\n */g, "\n")
+    .replace(/\n{2,}/g, "\n")
+    .trim();
+}