diff --git a/README.md b/README.md index e0c25f714c..ea19cf6b76 100644 --- a/README.md +++ b/README.md @@ -347,6 +347,8 @@ firecrawl https://example.com --format json -o data.json --pretty firecrawl scrape https://firecrawl.dev https://firecrawl.dev/blog https://docs.firecrawl.dev ``` +With an API key on the default Firecrawl API, a scrape also checks whether an Alexandria provider covers the page's site, sending only the host. When one does, a one-line hint on stderr names the provider and the `firecrawl list ` command for its tools, at most once per site per run. Keyless mode, a custom `--api-url`, `--domain-tools` and `--lockdown` skip the check. + #### Scrape Options | Option | Description | diff --git a/src/__tests__/commands/scrape.test.ts b/src/__tests__/commands/scrape.test.ts index b9c9606882..7dc544ce5a 100644 --- a/src/__tests__/commands/scrape.test.ts +++ b/src/__tests__/commands/scrape.test.ts @@ -7,6 +7,7 @@ import { executeScrape } from '../../commands/scrape'; import { getClient, isKeylessMode, keylessRequest } from '../../utils/client'; import { initializeConfig } from '../../utils/config'; import { setupTest, teardownTest } from '../utils/mock-client'; +import { resetSiteProviderHints } from '../../utils/alexandria-hint'; // Mock the Firecrawl client module vi.mock('../../utils/client', async () => { @@ -19,11 +20,22 @@ vi.mock('../../utils/client', async () => { }; }); +const resolveResponse = (skills: Array<{ id: string }>) => + new Response(JSON.stringify({ skills }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + describe('executeScrape', () => { let mockClient: any; + let fetchMock: ReturnType; beforeEach(() => { setupTest(); + resetSiteProviderHints(); + vi.mocked(isKeylessMode).mockReturnValue(false); + fetchMock = vi.fn(async () => resolveResponse([])); + vi.stubGlobal('fetch', fetchMock); // Initialize config with test API key initializeConfig({ apiKey: 'test-api-key', @@ -41,9 +53,166 @@ describe('executeScrape', () => { afterEach(() => { teardownTest(); + resetSiteProviderHints(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); vi.clearAllMocks(); }); + describe('Alexandria site hint', () => { + it('looks up the page host and prints a hint when a provider covers it', async () => { + fetchMock.mockResolvedValue(resolveResponse([{ id: 'fred' }])); + mockClient.scrape.mockResolvedValue({ markdown: '# Series' }); + const stderr = vi + .spyOn(process.stderr, 'write') + .mockImplementation(() => true); + + const result = await executeScrape({ + url: 'https://fred.stlouisfed.org/series/CPIAUCSL?x=1', + }); + + expect(result.success).toBe(true); + expect(fetchMock).toHaveBeenCalledTimes(1); + const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit]; + expect(url).toBe('https://api.firecrawl.dev/exchange/skills/resolve'); + expect(init.method).toBe('POST'); + expect(JSON.parse(init.body as string)).toEqual({ + urls: ['https://fred.stlouisfed.org/'], + }); + expect((init.headers as Record).Authorization).toBe( + 'Bearer test-api-key' + ); + const output = stderr.mock.calls.map(([text]) => String(text)).join(''); + expect(output).toContain( + 'Alexandria has a structured-data provider for fred.stlouisfed.org (fred).' + ); + expect(output).toContain('firecrawl list fred'); + expect(output).toContain('firecrawl scrape fred/'); + }); + + it('prints the hint once per host and looks the host up once', async () => { + fetchMock.mockResolvedValue(resolveResponse([{ id: 'fred' }])); + mockClient.scrape.mockResolvedValue({ markdown: '# Series' }); + const stderr = vi + .spyOn(process.stderr, 'write') + .mockImplementation(() => true); + + await executeScrape({ url: 'https://fred.stlouisfed.org/a' }); + await executeScrape({ url: 'https://fred.stlouisfed.org/b' }); + + expect(fetchMock).toHaveBeenCalledTimes(1); + const hints = stderr.mock.calls.filter(([text]) => + String(text).startsWith('Alexandria has') + ); + expect(hints).toHaveLength(1); + }); + + it('prints nothing when no provider matches or the lookup fails', async () => { + mockClient.scrape.mockResolvedValue({ markdown: '# Page' }); + const stderr = vi + .spyOn(process.stderr, 'write') + .mockImplementation(() => true); + + await executeScrape({ url: 'https://example.com' }); + fetchMock.mockResolvedValue(new Response('nope', { status: 503 })); + await executeScrape({ url: 'https://example.org' }); + fetchMock.mockRejectedValue(new Error('offline')); + const result = await executeScrape({ url: 'https://example.net' }); + + expect(result.success).toBe(true); + expect( + stderr.mock.calls.some(([text]) => + String(text).startsWith('Alexandria has') + ) + ).toBe(false); + }); + + it('skips the lookup with --domain-tools, lockdown, keyless mode or a self-hosted API', async () => { + mockClient.scrape.mockResolvedValue({ markdown: '# Page' }); + + await executeScrape({ url: 'https://example.com', domainTools: true }); + await executeScrape({ url: 'https://example.com', lockdown: true }); + await executeScrape({ + url: 'https://example.com', + apiUrl: 'http://localhost:3002', + }); + vi.mocked(isKeylessMode).mockReturnValue(true); + vi.mocked(keylessRequest).mockResolvedValueOnce({ + data: { markdown: '# Page' }, + }); + await executeScrape({ url: 'https://example.com' }); + + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('looks up the cloud API given with a trailing slash', async () => { + mockClient.scrape.mockResolvedValue({ markdown: '# Page' }); + + await executeScrape({ + url: 'https://example.com', + apiUrl: 'https://api.firecrawl.dev/', + }); + + expect(fetchMock).toHaveBeenCalledWith( + 'https://api.firecrawl.dev/exchange/skills/resolve', + expect.anything() + ); + }); + + it('keeps the hint for a concurrent same-host scrape when another fails', async () => { + let answer: (response: Response) => void = () => {}; + fetchMock.mockImplementation( + () => new Promise((resolve) => (answer = resolve)) + ); + let failScrape: (error: Error) => void = () => {}; + mockClient.scrape + .mockImplementationOnce( + () => new Promise((_resolve, reject) => (failScrape = reject)) + ) + .mockResolvedValueOnce({ markdown: '# Series' }); + const stderr = vi + .spyOn(process.stderr, 'write') + .mockImplementation(() => true); + + const failing = executeScrape({ url: 'https://fred.stlouisfed.org/a' }); + const succeeding = executeScrape({ + url: 'https://fred.stlouisfed.org/b', + }); + failScrape(new Error('boom')); + expect((await failing).success).toBe(false); + answer(resolveResponse([{ id: 'fred' }])); + expect((await succeeding).success).toBe(true); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect( + stderr.mock.calls.some(([text]) => + String(text).startsWith('Alexandria has') + ) + ).toBe(true); + }); + + it('aborts a late lookup instead of waiting for it', async () => { + let signal: AbortSignal | undefined; + fetchMock.mockImplementation( + (_url: string, init: RequestInit) => + new Promise((_resolve, reject) => { + signal = init.signal as AbortSignal; + signal.addEventListener('abort', () => + reject(new DOMException('Aborted', 'AbortError')) + ); + }) + ); + mockClient.scrape.mockResolvedValue({ markdown: '# Page' }); + + const started = Date.now(); + const result = await executeScrape({ url: 'https://example.com' }); + + expect(result.success).toBe(true); + expect(Date.now() - started).toBeLessThan(2_000); + expect(signal?.aborted).toBe(true); + }); + }); + describe('API call generation', () => { it('should call scrape with correct URL and default markdown format', async () => { const mockResponse = { markdown: '# Test Content' }; diff --git a/src/commands/scrape.ts b/src/commands/scrape.ts index 39b28d9cbc..dd1811ac95 100644 --- a/src/commands/scrape.ts +++ b/src/commands/scrape.ts @@ -20,6 +20,13 @@ import { executeMap } from './map'; import { getStatus } from './status'; import { requireAlexandriaKey } from './alexandria'; import { withoutToolPrice } from '../utils/alexandria-prices'; +import { + lookupSiteProviders, + releaseSiteProviderLookup, + settleSiteProviderHint, +} from '../utils/alexandria-hint'; + +const HINT_WAIT_MS = 500; /** * Output timing information if requested @@ -146,6 +153,11 @@ export async function executeScrape( // Execute scrape with timing - only wrap the scrape call in try-catch const requestStartTime = Date.now(); + const keyless = isKeylessMode(options.apiKey, options.apiUrl); + const siteLookup = + options.domainTools || options.lockdown || keyless + ? undefined + : lookupSiteProviders(options.url, options); try { if (options.toolDetail !== undefined) @@ -155,7 +167,7 @@ export async function executeScrape( scrapeParams.domainTools = true; } let result: any; - if (isKeylessMode(options.apiKey, options.apiUrl)) { + if (keyless) { // Keyless free tier: header-less request. The API identifies the CLI via // the `integration: 'cli'` field already in scrapeParams. const json = await keylessRequest('/v2/scrape', { @@ -190,6 +202,10 @@ export async function executeScrape( } } + const hint = + siteLookup && (await settleSiteProviderHint(siteLookup, HINT_WAIT_MS)); + if (hint) process.stderr.write(`${hint}\n`); + return { success: true, data: Array.isArray(result?.tools) @@ -197,6 +213,7 @@ export async function executeScrape( : result, }; } catch (error) { + if (siteLookup) releaseSiteProviderLookup(siteLookup); const requestEndTime = Date.now(); outputTiming(options, requestStartTime, requestEndTime, error); diff --git a/src/utils/alexandria-hint.ts b/src/utils/alexandria-hint.ts new file mode 100644 index 0000000000..ff3310ef51 --- /dev/null +++ b/src/utils/alexandria-hint.ts @@ -0,0 +1,150 @@ +/** + * Tells the user when Alexandria has a structured-data provider for a scraped + * page's site. The lookup sends only the host and never fails the scrape it + * accompanies. + */ + +import { getApiKey, getConfig } from './config'; +import { integrationHeaders } from './integration'; + +const CLOUD_API_URL = 'https://api.firecrawl.dev'; +const LOOKUP_TIMEOUT_MS = 5_000; +const PROVIDER_ID = /^[a-z0-9][a-z0-9._-]{0,99}$/i; + +type Lookup = { + host: string; + providers: Promise; + /** Scrapes still waiting on this lookup; the last one to let go of a running lookup aborts it. */ + waiting: number; + settled: boolean; + controller: AbortController; +}; + +const lookups = new Map(); +const hinted = new Set(); + +function hostOf(url: string): string | undefined { + try { + const parsed = new URL(url); + if (!['http:', 'https:'].includes(parsed.protocol)) return undefined; + return parsed.hostname.toLowerCase().replace(/\.$/, '') || undefined; + } catch { + return undefined; + } +} + +async function resolveProviders( + host: string, + apiKey: string, + apiUrl: string, + signal: AbortSignal +): Promise { + try { + const response = await fetch( + `${apiUrl.replace(/\/$/, '')}/exchange/skills/resolve`, + { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + ...integrationHeaders(), + }, + body: JSON.stringify({ urls: [`https://${host}/`] }), + signal: AbortSignal.any([ + signal, + AbortSignal.timeout(LOOKUP_TIMEOUT_MS), + ]), + redirect: 'error', + } + ); + if (!response.ok) return undefined; + const body: any = await response.json(); + if (!Array.isArray(body?.skills)) return undefined; + return [ + ...new Set( + body.skills + .map((skill: any) => skill?.id) + .filter( + (id: unknown): id is string => + typeof id === 'string' && PROVIDER_ID.test(id) + ) + ), + ]; + } catch { + return undefined; + } +} + +/** + * Starts (or joins) the provider lookup for a URL's host. Returns undefined + * when no lookup applies: no API key, a self-hosted API, or a non-web URL. + * Every lookup returned must be passed to `settleSiteProviderHint` or + * `releaseSiteProviderLookup`. + */ +export function lookupSiteProviders( + url: string, + options: { apiKey?: string; apiUrl?: string } +): Lookup | undefined { + const apiKey = getApiKey(options.apiKey); + const host = hostOf(url); + const apiUrl = ( + options.apiUrl || + getConfig().apiUrl || + CLOUD_API_URL + ).replace(/\/+$/, ''); + if (!apiKey || !host || apiUrl !== CLOUD_API_URL) return undefined; + const existing = lookups.get(host); + if (existing) { + existing.waiting++; + return existing; + } + const controller = new AbortController(); + const lookup: Lookup = { + host, + providers: resolveProviders(host, apiKey, apiUrl, controller.signal), + waiting: 1, + settled: false, + controller, + }; + void lookup.providers.then(() => { + lookup.settled = true; + }); + lookups.set(host, lookup); + return lookup; +} + +/** Stops waiting on a lookup, aborting it when no other scrape still waits on it. */ +export function releaseSiteProviderLookup(lookup: Lookup): void { + lookup.waiting--; + if (lookup.waiting > 0 || lookup.settled) return; + lookup.controller.abort(); + lookups.delete(lookup.host); +} + +/** + * Waits at most `waitMs` for a lookup started beside a scrape and returns its + * hint line, once per host per process. + */ +export async function settleSiteProviderHint( + lookup: Lookup, + waitMs: number +): Promise { + let timer: NodeJS.Timeout | undefined; + const late = new Promise<'late'>((resolve) => { + timer = setTimeout(() => resolve('late'), waitMs); + }); + const providers = await Promise.race([lookup.providers, late]); + clearTimeout(timer); + releaseSiteProviderLookup(lookup); + if (providers === 'late') return undefined; + const provider = providers?.[0]; + if (!provider || hinted.has(lookup.host)) return undefined; + hinted.add(lookup.host); + return `Alexandria has a structured-data provider for ${lookup.host} (${provider}). For typed records instead of page text, list its tools with \`firecrawl list ${provider}\`, then run one with \`firecrawl scrape ${provider}/ --options '{...}'\`.`; +} + +export function resetSiteProviderHints(): void { + for (const lookup of lookups.values()) lookup.controller.abort(); + lookups.clear(); + hinted.clear(); +}