From 6fba961ad02dfb6b91f557be961a50d4bfe7b4be Mon Sep 17 00:00:00 2001 From: nickscamara <20311743+nickscamara@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:57:58 +0000 Subject: [PATCH 1/2] feat(scrape): hint when an Alexandria provider covers the scraped site Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- README.md | 2 + src/__tests__/commands/scrape.test.ts | 123 ++++++++++++++++++++++++ src/commands/scrape.ts | 18 +++- src/utils/alexandria-hint.ts | 131 ++++++++++++++++++++++++++ 4 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 src/utils/alexandria-hint.ts diff --git a/README.md b/README.md index e0c25f714c..2bedf627aa 100644 --- a/README.md +++ b/README.md @@ -347,6 +347,8 @@ firecrawl https://example.com --format json -o data.json --pretty firecrawl scrape https://firecrawl.dev https://firecrawl.dev/blog https://docs.firecrawl.dev ``` +With an API key, a scrape also checks whether an Alexandria provider covers the page's site, sending only the host. When one does, a one-line hint on stderr names the provider and the `firecrawl list` command for its tools, at most once per site per run. `--domain-tools` and `--lockdown` skip the check. + #### Scrape Options | Option | Description | diff --git a/src/__tests__/commands/scrape.test.ts b/src/__tests__/commands/scrape.test.ts index b9c9606882..0a1151c3fd 100644 --- a/src/__tests__/commands/scrape.test.ts +++ b/src/__tests__/commands/scrape.test.ts @@ -7,6 +7,7 @@ import { executeScrape } from '../../commands/scrape'; import { getClient, isKeylessMode, keylessRequest } from '../../utils/client'; import { initializeConfig } from '../../utils/config'; import { setupTest, teardownTest } from '../utils/mock-client'; +import { resetSiteProviderHints } from '../../utils/alexandria-hint'; // Mock the Firecrawl client module vi.mock('../../utils/client', async () => { @@ -19,11 +20,22 @@ vi.mock('../../utils/client', async () => { }; }); +const resolveResponse = (skills: Array<{ id: string }>) => + new Response(JSON.stringify({ skills }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + describe('executeScrape', () => { let mockClient: any; + let fetchMock: ReturnType; beforeEach(() => { setupTest(); + resetSiteProviderHints(); + vi.mocked(isKeylessMode).mockReturnValue(false); + fetchMock = vi.fn(async () => resolveResponse([])); + vi.stubGlobal('fetch', fetchMock); // Initialize config with test API key initializeConfig({ apiKey: 'test-api-key', @@ -41,9 +53,120 @@ describe('executeScrape', () => { afterEach(() => { teardownTest(); + resetSiteProviderHints(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); vi.clearAllMocks(); }); + describe('Alexandria site hint', () => { + it('looks up the page host and prints a hint when a provider covers it', async () => { + fetchMock.mockResolvedValue(resolveResponse([{ id: 'fred' }])); + mockClient.scrape.mockResolvedValue({ markdown: '# Series' }); + const stderr = vi + .spyOn(process.stderr, 'write') + .mockImplementation(() => true); + + const result = await executeScrape({ + url: 'https://fred.stlouisfed.org/series/CPIAUCSL?x=1', + }); + + expect(result.success).toBe(true); + expect(fetchMock).toHaveBeenCalledTimes(1); + const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit]; + expect(url).toBe('https://api.firecrawl.dev/exchange/skills/resolve'); + expect(init.method).toBe('POST'); + expect(JSON.parse(init.body as string)).toEqual({ + urls: ['https://fred.stlouisfed.org/'], + }); + expect((init.headers as Record).Authorization).toBe( + 'Bearer test-api-key' + ); + const output = stderr.mock.calls.map(([text]) => String(text)).join(''); + expect(output).toContain( + 'Alexandria has a structured-data provider for fred.stlouisfed.org (fred).' + ); + expect(output).toContain('firecrawl list fred'); + expect(output).toContain('firecrawl scrape fred/'); + }); + + it('prints the hint once per host and looks the host up once', async () => { + fetchMock.mockResolvedValue(resolveResponse([{ id: 'fred' }])); + mockClient.scrape.mockResolvedValue({ markdown: '# Series' }); + const stderr = vi + .spyOn(process.stderr, 'write') + .mockImplementation(() => true); + + await executeScrape({ url: 'https://fred.stlouisfed.org/a' }); + await executeScrape({ url: 'https://fred.stlouisfed.org/b' }); + + expect(fetchMock).toHaveBeenCalledTimes(1); + const hints = stderr.mock.calls.filter(([text]) => + String(text).startsWith('Alexandria has') + ); + expect(hints).toHaveLength(1); + }); + + it('prints nothing when no provider matches or the lookup fails', async () => { + mockClient.scrape.mockResolvedValue({ markdown: '# Page' }); + const stderr = vi + .spyOn(process.stderr, 'write') + .mockImplementation(() => true); + + await executeScrape({ url: 'https://example.com' }); + fetchMock.mockResolvedValue(new Response('nope', { status: 503 })); + await executeScrape({ url: 'https://example.org' }); + fetchMock.mockRejectedValue(new Error('offline')); + const result = await executeScrape({ url: 'https://example.net' }); + + expect(result.success).toBe(true); + expect( + stderr.mock.calls.some(([text]) => + String(text).startsWith('Alexandria has') + ) + ).toBe(false); + }); + + it('skips the lookup with --domain-tools, lockdown, keyless mode or a self-hosted API', async () => { + mockClient.scrape.mockResolvedValue({ markdown: '# Page' }); + + await executeScrape({ url: 'https://example.com', domainTools: true }); + await executeScrape({ url: 'https://example.com', lockdown: true }); + await executeScrape({ + url: 'https://example.com', + apiUrl: 'http://localhost:3002', + }); + vi.mocked(isKeylessMode).mockReturnValue(true); + vi.mocked(keylessRequest).mockResolvedValueOnce({ + data: { markdown: '# Page' }, + }); + await executeScrape({ url: 'https://example.com' }); + + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('aborts a late lookup instead of waiting for it', async () => { + let signal: AbortSignal | undefined; + fetchMock.mockImplementation( + (_url: string, init: RequestInit) => + new Promise((_resolve, reject) => { + signal = init.signal as AbortSignal; + signal.addEventListener('abort', () => + reject(new DOMException('Aborted', 'AbortError')) + ); + }) + ); + mockClient.scrape.mockResolvedValue({ markdown: '# Page' }); + + const started = Date.now(); + const result = await executeScrape({ url: 'https://example.com' }); + + expect(result.success).toBe(true); + expect(Date.now() - started).toBeLessThan(2_000); + expect(signal?.aborted).toBe(true); + }); + }); + describe('API call generation', () => { it('should call scrape with correct URL and default markdown format', async () => { const mockResponse = { markdown: '# Test Content' }; diff --git a/src/commands/scrape.ts b/src/commands/scrape.ts index 39b28d9cbc..5c5ca1f797 100644 --- a/src/commands/scrape.ts +++ b/src/commands/scrape.ts @@ -20,6 +20,12 @@ import { executeMap } from './map'; import { getStatus } from './status'; import { requireAlexandriaKey } from './alexandria'; import { withoutToolPrice } from '../utils/alexandria-prices'; +import { + lookupSiteProviders, + settleSiteProviderHint, +} from '../utils/alexandria-hint'; + +const HINT_WAIT_MS = 500; /** * Output timing information if requested @@ -146,6 +152,11 @@ export async function executeScrape( // Execute scrape with timing - only wrap the scrape call in try-catch const requestStartTime = Date.now(); + const keyless = isKeylessMode(options.apiKey, options.apiUrl); + const siteLookup = + options.domainTools || options.lockdown || keyless + ? undefined + : lookupSiteProviders(options.url, options); try { if (options.toolDetail !== undefined) @@ -155,7 +166,7 @@ export async function executeScrape( scrapeParams.domainTools = true; } let result: any; - if (isKeylessMode(options.apiKey, options.apiUrl)) { + if (keyless) { // Keyless free tier: header-less request. The API identifies the CLI via // the `integration: 'cli'` field already in scrapeParams. const json = await keylessRequest('/v2/scrape', { @@ -190,6 +201,10 @@ export async function executeScrape( } } + const hint = + siteLookup && (await settleSiteProviderHint(siteLookup, HINT_WAIT_MS)); + if (hint) process.stderr.write(`${hint}\n`); + return { success: true, data: Array.isArray(result?.tools) @@ -197,6 +212,7 @@ export async function executeScrape( : result, }; } catch (error) { + siteLookup?.abort(); const requestEndTime = Date.now(); outputTiming(options, requestStartTime, requestEndTime, error); diff --git a/src/utils/alexandria-hint.ts b/src/utils/alexandria-hint.ts new file mode 100644 index 0000000000..eee3545d1d --- /dev/null +++ b/src/utils/alexandria-hint.ts @@ -0,0 +1,131 @@ +/** + * Tells the user when Alexandria has a structured-data provider for a scraped + * page's site. The lookup sends only the host and never fails the scrape it + * accompanies. + */ + +import { getApiKey, getConfig, isCustomApiUrl } from './config'; +import { integrationHeaders } from './integration'; + +const LOOKUP_TIMEOUT_MS = 5_000; +const PROVIDER_ID = /^[a-z0-9][a-z0-9._-]{0,99}$/i; + +type Lookup = { + host: string; + providers: Promise; + /** Stops a lookup still running, so it never keeps the process alive. */ + abort(): void; +}; + +const lookups = new Map(); +const hinted = new Set(); + +function hostOf(url: string): string | undefined { + try { + const parsed = new URL(url); + if (!['http:', 'https:'].includes(parsed.protocol)) return undefined; + return parsed.hostname.toLowerCase().replace(/\.$/, '') || undefined; + } catch { + return undefined; + } +} + +async function resolveProviders( + host: string, + apiKey: string, + apiUrl: string, + signal: AbortSignal +): Promise { + try { + const response = await fetch( + `${apiUrl.replace(/\/$/, '')}/exchange/skills/resolve`, + { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + ...integrationHeaders(), + }, + body: JSON.stringify({ urls: [`https://${host}/`] }), + signal: AbortSignal.any([ + signal, + AbortSignal.timeout(LOOKUP_TIMEOUT_MS), + ]), + redirect: 'error', + } + ); + if (!response.ok) return undefined; + const body: any = await response.json(); + if (!Array.isArray(body?.skills)) return undefined; + return [ + ...new Set( + body.skills + .map((skill: any) => skill?.id) + .filter( + (id: unknown): id is string => + typeof id === 'string' && PROVIDER_ID.test(id) + ) + ), + ]; + } catch { + return undefined; + } +} + +/** + * Starts (or reuses) the provider lookup for a URL's host. Returns undefined + * when no lookup applies: no API key, a self-hosted API, or a non-web URL. + */ +export function lookupSiteProviders( + url: string, + options: { apiKey?: string; apiUrl?: string } +): Lookup | undefined { + const apiKey = getApiKey(options.apiKey); + const host = hostOf(url); + if (!apiKey || !host || isCustomApiUrl(options.apiUrl)) return undefined; + const existing = lookups.get(host); + if (existing) return existing; + const controller = new AbortController(); + const lookup: Lookup = { + host, + providers: resolveProviders( + host, + apiKey, + options.apiUrl || getConfig().apiUrl || 'https://api.firecrawl.dev', + controller.signal + ), + abort: () => controller.abort(), + }; + lookups.set(host, lookup); + return lookup; +} + +/** + * Waits at most `waitMs` for a lookup started beside a scrape and returns its + * hint line, once per host per process. A late lookup is aborted. + */ +export async function settleSiteProviderHint( + lookup: Lookup, + waitMs: number +): Promise { + let timer: NodeJS.Timeout | undefined; + const late = new Promise<'late'>((resolve) => { + timer = setTimeout(() => resolve('late'), waitMs); + }); + const providers = await Promise.race([lookup.providers, late]); + clearTimeout(timer); + if (providers === 'late') { + lookup.abort(); + return undefined; + } + const provider = providers?.[0]; + if (!provider || hinted.has(lookup.host)) return undefined; + hinted.add(lookup.host); + return `Alexandria has a structured-data provider for ${lookup.host} (${provider}). For typed records instead of page text, list its tools with \`firecrawl list ${provider}\`, then run one with \`firecrawl scrape ${provider}/ --options '{...}'\`.`; +} + +export function resetSiteProviderHints(): void { + for (const lookup of lookups.values()) lookup.abort(); + lookups.clear(); + hinted.clear(); +} From 8aabc9799cd5645f479df348138f44956916cd3f Mon Sep 17 00:00:00 2001 From: nickscamara <20311743+nickscamara@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:02:48 +0000 Subject: [PATCH 2/2] fix(scrape): keep shared site lookups alive for concurrent scrapes; accept the cloud URL with a trailing slash Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- README.md | 2 +- src/__tests__/commands/scrape.test.ts | 46 +++++++++++++++++++++ src/commands/scrape.ts | 3 +- src/utils/alexandria-hint.ts | 57 ++++++++++++++++++--------- 4 files changed, 87 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 2bedf627aa..ea19cf6b76 100644 --- a/README.md +++ b/README.md @@ -347,7 +347,7 @@ firecrawl https://example.com --format json -o data.json --pretty firecrawl scrape https://firecrawl.dev https://firecrawl.dev/blog https://docs.firecrawl.dev ``` -With an API key, a scrape also checks whether an Alexandria provider covers the page's site, sending only the host. When one does, a one-line hint on stderr names the provider and the `firecrawl list` command for its tools, at most once per site per run. `--domain-tools` and `--lockdown` skip the check. +With an API key on the default Firecrawl API, a scrape also checks whether an Alexandria provider covers the page's site, sending only the host. When one does, a one-line hint on stderr names the provider and the `firecrawl list ` command for its tools, at most once per site per run. Keyless mode, a custom `--api-url`, `--domain-tools` and `--lockdown` skip the check. #### Scrape Options diff --git a/src/__tests__/commands/scrape.test.ts b/src/__tests__/commands/scrape.test.ts index 0a1151c3fd..7dc544ce5a 100644 --- a/src/__tests__/commands/scrape.test.ts +++ b/src/__tests__/commands/scrape.test.ts @@ -145,6 +145,52 @@ describe('executeScrape', () => { expect(fetchMock).not.toHaveBeenCalled(); }); + it('looks up the cloud API given with a trailing slash', async () => { + mockClient.scrape.mockResolvedValue({ markdown: '# Page' }); + + await executeScrape({ + url: 'https://example.com', + apiUrl: 'https://api.firecrawl.dev/', + }); + + expect(fetchMock).toHaveBeenCalledWith( + 'https://api.firecrawl.dev/exchange/skills/resolve', + expect.anything() + ); + }); + + it('keeps the hint for a concurrent same-host scrape when another fails', async () => { + let answer: (response: Response) => void = () => {}; + fetchMock.mockImplementation( + () => new Promise((resolve) => (answer = resolve)) + ); + let failScrape: (error: Error) => void = () => {}; + mockClient.scrape + .mockImplementationOnce( + () => new Promise((_resolve, reject) => (failScrape = reject)) + ) + .mockResolvedValueOnce({ markdown: '# Series' }); + const stderr = vi + .spyOn(process.stderr, 'write') + .mockImplementation(() => true); + + const failing = executeScrape({ url: 'https://fred.stlouisfed.org/a' }); + const succeeding = executeScrape({ + url: 'https://fred.stlouisfed.org/b', + }); + failScrape(new Error('boom')); + expect((await failing).success).toBe(false); + answer(resolveResponse([{ id: 'fred' }])); + expect((await succeeding).success).toBe(true); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect( + stderr.mock.calls.some(([text]) => + String(text).startsWith('Alexandria has') + ) + ).toBe(true); + }); + it('aborts a late lookup instead of waiting for it', async () => { let signal: AbortSignal | undefined; fetchMock.mockImplementation( diff --git a/src/commands/scrape.ts b/src/commands/scrape.ts index 5c5ca1f797..dd1811ac95 100644 --- a/src/commands/scrape.ts +++ b/src/commands/scrape.ts @@ -22,6 +22,7 @@ import { requireAlexandriaKey } from './alexandria'; import { withoutToolPrice } from '../utils/alexandria-prices'; import { lookupSiteProviders, + releaseSiteProviderLookup, settleSiteProviderHint, } from '../utils/alexandria-hint'; @@ -212,7 +213,7 @@ export async function executeScrape( : result, }; } catch (error) { - siteLookup?.abort(); + if (siteLookup) releaseSiteProviderLookup(siteLookup); const requestEndTime = Date.now(); outputTiming(options, requestStartTime, requestEndTime, error); diff --git a/src/utils/alexandria-hint.ts b/src/utils/alexandria-hint.ts index eee3545d1d..ff3310ef51 100644 --- a/src/utils/alexandria-hint.ts +++ b/src/utils/alexandria-hint.ts @@ -4,17 +4,20 @@ * accompanies. */ -import { getApiKey, getConfig, isCustomApiUrl } from './config'; +import { getApiKey, getConfig } from './config'; import { integrationHeaders } from './integration'; +const CLOUD_API_URL = 'https://api.firecrawl.dev'; const LOOKUP_TIMEOUT_MS = 5_000; const PROVIDER_ID = /^[a-z0-9][a-z0-9._-]{0,99}$/i; type Lookup = { host: string; providers: Promise; - /** Stops a lookup still running, so it never keeps the process alive. */ - abort(): void; + /** Scrapes still waiting on this lookup; the last one to let go of a running lookup aborts it. */ + waiting: number; + settled: boolean; + controller: AbortController; }; const lookups = new Map(); @@ -73,8 +76,10 @@ async function resolveProviders( } /** - * Starts (or reuses) the provider lookup for a URL's host. Returns undefined + * Starts (or joins) the provider lookup for a URL's host. Returns undefined * when no lookup applies: no API key, a self-hosted API, or a non-web URL. + * Every lookup returned must be passed to `settleSiteProviderHint` or + * `releaseSiteProviderLookup`. */ export function lookupSiteProviders( url: string, @@ -82,27 +87,43 @@ export function lookupSiteProviders( ): Lookup | undefined { const apiKey = getApiKey(options.apiKey); const host = hostOf(url); - if (!apiKey || !host || isCustomApiUrl(options.apiUrl)) return undefined; + const apiUrl = ( + options.apiUrl || + getConfig().apiUrl || + CLOUD_API_URL + ).replace(/\/+$/, ''); + if (!apiKey || !host || apiUrl !== CLOUD_API_URL) return undefined; const existing = lookups.get(host); - if (existing) return existing; + if (existing) { + existing.waiting++; + return existing; + } const controller = new AbortController(); const lookup: Lookup = { host, - providers: resolveProviders( - host, - apiKey, - options.apiUrl || getConfig().apiUrl || 'https://api.firecrawl.dev', - controller.signal - ), - abort: () => controller.abort(), + providers: resolveProviders(host, apiKey, apiUrl, controller.signal), + waiting: 1, + settled: false, + controller, }; + void lookup.providers.then(() => { + lookup.settled = true; + }); lookups.set(host, lookup); return lookup; } +/** Stops waiting on a lookup, aborting it when no other scrape still waits on it. */ +export function releaseSiteProviderLookup(lookup: Lookup): void { + lookup.waiting--; + if (lookup.waiting > 0 || lookup.settled) return; + lookup.controller.abort(); + lookups.delete(lookup.host); +} + /** * Waits at most `waitMs` for a lookup started beside a scrape and returns its - * hint line, once per host per process. A late lookup is aborted. + * hint line, once per host per process. */ export async function settleSiteProviderHint( lookup: Lookup, @@ -114,10 +135,8 @@ export async function settleSiteProviderHint( }); const providers = await Promise.race([lookup.providers, late]); clearTimeout(timer); - if (providers === 'late') { - lookup.abort(); - return undefined; - } + releaseSiteProviderLookup(lookup); + if (providers === 'late') return undefined; const provider = providers?.[0]; if (!provider || hinted.has(lookup.host)) return undefined; hinted.add(lookup.host); @@ -125,7 +144,7 @@ export async function settleSiteProviderHint( } export function resetSiteProviderHints(): void { - for (const lookup of lookups.values()) lookup.abort(); + for (const lookup of lookups.values()) lookup.controller.abort(); lookups.clear(); hinted.clear(); }