feat: add fal.ai generative media tool to the assistive model

src/chrome/src/agent/agent.js

@@ -124,6 +124,7 @@ import {
   visionGenerationOptions,
 } from '../providers/provider-compatibility.js';
 import { resolveMaxOutputTokens } from '../providers/context-windows.js';
+import { generateImage } from './fal-media.js';
 import { extractFirstJsonObject } from './json-extract.js';
 import { repairAssistantDisplayText, sanitizeText as sanitizePlannerText } from './text-sanitize.js';
 import { emptyOutputFailureMessage, modelOutputDiagnostics } from './model-output-diagnostics.js';
@@ -35600,6 +35601,9 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
     if (name === 'fetch_url') {
       return await fetchUrl(args.url, args, { tabId, signal: executionContext?._contentActionAbortSignal });
     }
+    if (name === 'generate_image') {
+      return await generateImage(args, { signal: executionContext?._contentActionAbortSignal });
+    }
     if (name === 'read_page_source') {
       return await readPageSource(args.url, args, { tabId, signal: executionContext?._contentActionAbortSignal });
     }

src/chrome/src/agent/fal-media.js

@@ -0,0 +1,270 @@
+// fal.ai generative media (assistive model).
+//
+// Configured in Settings -> Assistive Models -> "Generative media (fal.ai)"
+// and stored in extension storage as `imageGenModel = { apiKey, model }`.
+// fal.ai uses a queue API: submit a prompt, poll status_url, then fetch
+// response_url. Auth uses `Authorization: Key <FAL_KEY>`.
+
+export const IMAGE_GEN_MODEL_KEY = 'imageGenModel';
+export const FAL_QUEUE_BASE = 'https://queue.fal.run';
+export const FAL_AUTH_PROBE_URL = 'https://api.fal.ai/v1/workflows?limit=1';
+const FAL_STATUS_POLL_INTERVAL_MS = 2000;
+const FAL_STATUS_TIMEOUT_MS = 120000;
+const FAL_CANCEL_TIMEOUT_MS = 2000;
+
+/** Normalize a fal.ai model id (for example, "fal-ai/flux/schnell"). */
+export function normalizeFalModelId(model) {
+  const id = String(model || '').trim().replace(/^\/+|\/+$/g, '');
+  if (!id) return '';
+  if (!/^[A-Za-z0-9][A-Za-z0-9._\-/]*$/.test(id) || id.includes('..')) return '';
+  return id;
+}
+
+export function isImageGenConfigured(cfg) {
+  return !!(cfg && cfg.apiKey && cfg.model);
+}
+
+export function falQueueSubmitUrl(model) {
+  return `${FAL_QUEUE_BASE}/${model}`;
+}
+
+/** Extract a usable media URL from the response shapes used by fal models. */
+export function extractFalMediaUrl(payload) {
+  if (!payload || typeof payload !== 'object') return '';
+  if (typeof payload.url === 'string' && /^https:\/\//.test(payload.url)) return payload.url;
+  for (const listKey of ['images', 'videos', 'audio']) {
+    const list = payload[listKey];
+    if (Array.isArray(list) && typeof list[0]?.url === 'string' && /^https:\/\//.test(list[0].url)) {
+      return list[0].url;
+    }
+  }
+  for (const objKey of ['image', 'video', 'audio']) {
+    const obj = payload[objKey];
+    if (obj && typeof obj.url === 'string' && /^https:\/\//.test(obj.url)) return obj.url;
+  }
+  return '';
+}
+
+async function falAuthHeaders(apiKey) {
+  return { 'Authorization': `Key ${apiKey}`, 'Content-Type': 'application/json' };
+}
+
+function trustedFalQueueUrl(value) {
+  if (typeof value !== 'string' || !value) return '';
+  try {
+    const url = new URL(value);
+    return url.protocol === 'https:' && url.origin === FAL_QUEUE_BASE ? url.href : '';
+  } catch {
+    return '';
+  }
+}
+
+function abortReason(signal) {
+  if (signal?.reason instanceof Error) return signal.reason;
+  const error = new Error('fal.ai generation was cancelled.');
+  error.name = 'AbortError';
+  return error;
+}
+
+function throwIfAborted(signal) {
+  if (signal?.aborted) throw abortReason(signal);
+}
+
+function abortableDelay(ms, signal) {
+  throwIfAborted(signal);
+  return new Promise((resolve, reject) => {
+    const onAbort = () => {
+      clearTimeout(timer);
+      reject(abortReason(signal));
+    };
+    const timer = setTimeout(() => {
+      signal?.removeEventListener?.('abort', onAbort);
+      resolve();
+    }, ms);
+    signal?.addEventListener?.('abort', onAbort, { once: true });
+  });
+}
+
+function createOperationSignal(externalSignal, timeoutMs) {
+  const controller = new AbortController();
+  const timeoutError = new Error('fal.ai generation timed out.');
+  timeoutError.name = 'TimeoutError';
+  let timedOut = false;
+  const onExternalAbort = () => {
+    if (!controller.signal.aborted) controller.abort(abortReason(externalSignal));
+  };
+  if (externalSignal?.aborted) onExternalAbort();
+  else externalSignal?.addEventListener?.('abort', onExternalAbort, { once: true });
+  const timer = setTimeout(() => {
+    timedOut = true;
+    if (!controller.signal.aborted) controller.abort(timeoutError);
+  }, timeoutMs);
+  return {
+    signal: controller.signal,
+    timeoutError,
+    timedOut: () => timedOut,
+    dispose() {
+      clearTimeout(timer);
+      externalSignal?.removeEventListener?.('abort', onExternalAbort);
+    },
+  };
+}
+
+async function cancelFalRequest(cancelUrl, headers, fetchImpl) {
+  if (!cancelUrl) return;
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), FAL_CANCEL_TIMEOUT_MS);
+  try {
+    await fetchImpl(cancelUrl, {
+      method: 'PUT',
+      headers,
+      signal: controller.signal,
+      keepalive: true,
+    });
+  } catch {
+    // Cancellation is best-effort; preserve the original abort/timeout error.
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/** Run a queued generation: submit, poll, then fetch the result. */
+export async function runFalGeneration({
+  prompt,
+  config,
+  fetchImpl = fetch,
+  timeoutMs = FAL_STATUS_TIMEOUT_MS,
+  signal = null,
+}) {
+  const model = normalizeFalModelId(config?.model);
+  if (!model) throw new Error('Invalid fal.ai model id.');
+  if (!config?.apiKey) throw new Error('fal.ai API key not configured.');
+  const text = String(prompt || '').trim();
+  if (!text) throw new Error('prompt is required.');
+
+  const operation = createOperationSignal(signal, timeoutMs);
+  const headers = await falAuthHeaders(config.apiKey);
+  let cancelUrl = '';
+  let completed = false;
+  try {
+    throwIfAborted(operation.signal);
+    const submitRes = await fetchImpl(falQueueSubmitUrl(model), {
+      method: 'POST',
+      headers,
+      body: JSON.stringify({ prompt: text }),
+      signal: operation.signal,
+    });
+    if (!submitRes.ok) {
+      let body = '';
+      try { body = (await submitRes.text()).slice(0, 300); } catch { /* ignore */ }
+      throw new Error(`fal.ai submit failed (HTTP ${submitRes.status}): ${body || submitRes.statusText}`);
+    }
+
+    let queued;
+    try {
+      queued = await submitRes.json();
+    } catch (error) {
+      throw new Error(`fal.ai submit returned invalid JSON: ${error.message}`);
+    }
+    const statusUrl = trustedFalQueueUrl(queued?.status_url);
+    const responseUrl = trustedFalQueueUrl(queued?.response_url);
+    cancelUrl = trustedFalQueueUrl(queued?.cancel_url)
+      || (responseUrl ? `${responseUrl.replace(/\/$/, '')}/cancel` : '');
+    if (!statusUrl || !responseUrl) {
+      throw new Error('fal.ai submit response missing trusted status_url/response_url.');
+    }
+
+    let status = 'IN_QUEUE';
+    while (true) {
+      await abortableDelay(FAL_STATUS_POLL_INTERVAL_MS, operation.signal);
+      const statusRes = await fetchImpl(statusUrl, { headers, signal: operation.signal });
+      if (!statusRes.ok) throw new Error(`fal.ai status check failed (HTTP ${statusRes.status}).`);
+
+      let statusPayload;
+      try {
+        statusPayload = await statusRes.json();
+      } catch (error) {
+        throw new Error(`fal.ai status returned invalid JSON: ${error.message}`);
+      }
+      status = String(statusPayload?.status || '').toUpperCase();
+      if (status === 'COMPLETED') {
+        const resultRes = await fetchImpl(responseUrl, { headers, signal: operation.signal });
+        if (!resultRes.ok) throw new Error(`fal.ai result fetch failed (HTTP ${resultRes.status}).`);
+
+        let payload;
+        try {
+          payload = await resultRes.json();
+        } catch (error) {
+          throw new Error(`fal.ai result returned invalid JSON: ${error.message}`);
+        }
+        const url = extractFalMediaUrl(payload);
+        if (!url) throw new Error('fal.ai result contained no media URL.');
+        completed = true;
+        return { url, model, status };
+      }
+      if (status === 'FAILED' || status === 'ERROR') {
+        const errorText = typeof statusPayload?.error === 'string' ? statusPayload.error : 'unknown error';
+        throw new Error(`fal.ai generation failed: ${errorText}`);
+      }
+    }
+  } catch (error) {
+    if (cancelUrl && !completed) await cancelFalRequest(cancelUrl, headers, fetchImpl);
+    if (operation.timedOut()) throw operation.timeoutError;
+    if (signal?.aborted) throw abortReason(signal);
+    throw error;
+  } finally {
+    operation.dispose();
+  }
+}
+
+/** Agent tool entry point. Reads the assistive-model config from storage. */
+export async function generateImage(args, options = {}) {
+  const fetchImpl = typeof options === 'function' ? options : (options.fetchImpl || fetch);
+  const signal = typeof options === 'object' ? options.signal : null;
+  let cfg;
+  const api = (typeof browser !== 'undefined' && browser?.storage) ? browser
+    : (typeof chrome !== 'undefined' ? chrome : null);
+  try {
+    const stored = await api.storage.local.get([IMAGE_GEN_MODEL_KEY]);
+    cfg = stored?.[IMAGE_GEN_MODEL_KEY];
+  } catch (error) {
+    return { success: false, error: 'Failed to read generative media config: ' + error.message };
+  }
+  if (!isImageGenConfigured(cfg)) {
+    return { success: false, error: 'Generative media is not configured. Set up fal.ai in Settings -> Assistive Models.' };
+  }
+  try {
+    const result = await runFalGeneration({ prompt: args?.prompt, config: cfg, fetchImpl, signal });
+    return { success: true, url: result.url, model: result.model };
+  } catch (error) {
+    return { success: false, error: error.message };
+  }
+}
+
+/** Verify a key against a free, read-only endpoint that requires authentication. */
+export async function testImageGenProvider(fetchImpl = fetch) {
+  let cfg;
+  const api = (typeof browser !== 'undefined' && browser?.storage) ? browser
+    : (typeof chrome !== 'undefined' ? chrome : null);
+  try {
+    const stored = await api.storage.local.get([IMAGE_GEN_MODEL_KEY]);
+    cfg = stored?.[IMAGE_GEN_MODEL_KEY];
+  } catch (error) {
+    return { ok: false, error: 'Failed to read generative media config: ' + error.message };
+  }
+  if (!isImageGenConfigured(cfg)) {
+    return { ok: false, error: 'Generative media not configured (API Key and Model are required).' };
+  }
+  const model = normalizeFalModelId(cfg.model);
+  if (!model) return { ok: false, error: 'Invalid fal.ai model id.' };
+  try {
+    const res = await fetchImpl(FAL_AUTH_PROBE_URL, { headers: await falAuthHeaders(cfg.apiKey) });
+    if (res.status === 401 || res.status === 403) {
+      return { ok: false, error: 'fal.ai rejected the API key (HTTP ' + res.status + ').' };
+    }
+    if (res.ok) return { ok: true, model };
+    return { ok: false, error: `Unexpected response from fal.ai (HTTP ${res.status}).` };
+  } catch (error) {
+    return { ok: false, error: error.message };
+  }
+}

src/chrome/src/agent/permission-gate.js

@@ -82,6 +82,8 @@ export const UNTRUSTED_CONTENT_TOOLS = new Set([
   'execute_webmcp_tool',
   'fetch_url',
   'research_url',
+  // fal.ai returns provider-authored URLs and error text.
+  'generate_image',
   // ChatGPT's answer and cited links are third-party page content.
   'delegate_research',
   'read_pdf',
@@ -419,6 +421,8 @@ const TOOL_CAPABILITY = {
   download_social_media: Capability.DOWNLOAD,
   schedule_resume: Capability.SCHEDULE,
   schedule_task: Capability.SCHEDULE,
+  // generate_image spends the user's fal.ai credits via a paid network call.
+  generate_image: Capability.NETWORK,
 };
 
 /**
@@ -564,6 +568,7 @@ export function hostForCapability(capability, args, currentUrlOrHost, toolName)
     return normalizeHost(args._otpMailboxUrl);
   }
   if (toolName === 'delegate_research') return 'chatgpt.com';
+  if (toolName === 'generate_image') return 'queue.fal.run';
   if (toolName === 'execute_webmcp_tool') {
     // A tool can belong to a cross-origin frame. Charge mutations to that
     // frame's resolved URL instead of borrowing the top-level page grant.

src/chrome/src/agent/planner.js

@@ -376,6 +376,7 @@ ${PLANNER_RESPONSE_LANGUAGE_RULES}
   read: get_accessibility_tree, read_page, extract_data, fetch_url, research_url
   interact: click_ax, set_checked, type_ax, set_field, find_text, press_keys, scroll, navigate, gmail_count_results, carousel_navigate, promote_iframe
   wait: wait_for_element, wait_for_stable
+  media: generate_image (create an image/video/audio directly from a text prompt via the user's fal.ai generative-media model — use when the user asks to GENERATE media, not to browse an image site)
   memory: scratchpad_write, progress_update, progress_read
   schedule: schedule_task (future/recurring work the user explicitly asked for), schedule_resume (pause CURRENT run blocked on external event)
   user input: clarify (pause and ask one concise question when a required value remains missing after relevant inspection)
@@ -464,6 +465,7 @@ ${PLANNER_RESPONSE_ONLY_RULES}
 - Canonical summary, steps, and risks must be English. localized fields must use the requested wbLocale.
 ${PLANNER_RESPONSE_LANGUAGE_RULES}
 - For execute, keep the compact plan to 1–4 steps. For plan_only, provide 2–8 useful steps. For respond and clarify, steps may be empty.
+- When the user asks to generate an image/video/audio, plan one generate_image step (WebBrain's built-in fal.ai media tool). Do not plan steps to visit image-generation sites or to check whether the current page supports image generation.
 - clarify pauses execution to ask one concise question for a required value. done is terminal and must never be used to request information needed to continue.
 - press_keys supports only unmodified Escape, Tab, Enter, arrow keys, and ; (semicolon, for page shortcuts such as Gmail Expand all). Never plan modifier combinations or browser UI shortcuts; use find_text to select one page-text match instead of Ctrl/Cmd+F. Each call replaces the previous selection and cannot create simultaneous highlights or browser Find UI.
 - For Instagram /p/<id>/ carousel enumeration, use strictly increasing carousel_navigate indexes unless the latest user request explicitly asks for reverse traversal, in which case use strictly decreasing indexes; never use arrow keys, coordinate clicks, Previous/Next, or go_back to traverse slides.

src/chrome/src/agent/tools.js

@@ -1244,6 +1244,20 @@ export const AGENT_TOOLS = [
       },
     },
   },
+  {
+    type: 'function',
+    function: {
+      name: 'generate_image',
+      description: 'Generate media (usually an image) from a text prompt using the user\'s configured fal.ai assistive model (Settings → Assistive Models → Generative media). Runs on fal.ai\'s queue API and may take up to a minute. Returns the hosted media URL on success. Not available in Ask mode.',
+      parameters: {
+        type: 'object',
+        properties: {
+          prompt: { type: 'string', description: 'Text prompt describing the media to generate.' },
+        },
+        required: ['prompt'],
+      },
+    },
+  },
 ];
 
 /**
@@ -1952,6 +1966,7 @@ ${BROWSER_TAB_LIMITATION}
 - scratchpad_write: Pin a note in context that survives summarization (use on long tasks to remember download IDs, file paths, plans)
 - progress_update / progress_read: Structured app-owned ledger for the active repeated item/action task. Use it for per-user/per-item status and collected fields; close pending/acted rows before done.
 - download_public_media (if enabled by a skill) / download_social_media: One-shot image/video download from public social sites. Prefer the enabled skill tool for public media URLs; otherwise use download_social_media. Single call — no need to inspect the DOM yourself.
+- generate_image: Create media (usually an image, sometimes video/audio) directly from a text prompt through the user's configured fal.ai generative-media model. When the user asks to GENERATE media ("generate an image of a red apple", "make a logo", "create a video clip"), call this tool — do NOT navigate to third-party image sites (Midjourney, DALL·E, Bing Images, etc.). Requires Settings → Assistive Models → Generative media. Not available in Ask mode.
 - Recording is user-driven only. If the user asks to record, do NOT call tools; tell them to type \`/record\` for current-tab recording or \`/record --full-screen\` for screen/window recording; add \`--transcribe\` to either form if they want a Whisper transcript after stop. If they ask to stop a recording, tell them to press Escape twice in WebBrain/browser surfaces or use Chrome's Stop sharing control.
 - hover: CDP-trusted hover over a ref_id. Use ONLY for menus/tooltips that REVEAL on hover (GitHub three-dot menus, Linear card actions, nav menus with reveal-on-hover children). Re-read the tree after to find the newly-visible items. Do NOT call hover before every click — most things are clickable directly.
 - drag_drop: Drag one ref_id onto another via CDP-trusted pointer events. Use for Trello/Linear/Notion-style card reordering, file-tree node moves, image-crop handles, slider thumbs. Pass \`steps: 15–20\` if the first attempt doesn't trigger the drop indicator on momentum-tracking dnd. Verify by re-reading the tree.