Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,8 @@ firecrawl https://example.com --format json -o data.json --pretty
firecrawl scrape https://firecrawl.dev https://firecrawl.dev/blog https://docs.firecrawl.dev
```

With an API key on the default Firecrawl API, a scrape also checks whether an Alexandria provider covers the page's site, sending only the host. When one does, a one-line hint on stderr names the provider and the `firecrawl list <provider>` command for its tools, at most once per site per run. Keyless mode, a custom `--api-url`, `--domain-tools` and `--lockdown` skip the check.

#### Scrape Options

| Option | Description |
Expand Down
169 changes: 169 additions & 0 deletions src/__tests__/commands/scrape.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { executeScrape } from '../../commands/scrape';
import { getClient, isKeylessMode, keylessRequest } from '../../utils/client';
import { initializeConfig } from '../../utils/config';
import { setupTest, teardownTest } from '../utils/mock-client';
import { resetSiteProviderHints } from '../../utils/alexandria-hint';

// Mock the Firecrawl client module
vi.mock('../../utils/client', async () => {
Expand All @@ -19,11 +20,22 @@ vi.mock('../../utils/client', async () => {
};
});

const resolveResponse = (skills: Array<{ id: string }>) =>
new Response(JSON.stringify({ skills }), {
status: 200,
headers: { 'content-type': 'application/json' },
});

describe('executeScrape', () => {
let mockClient: any;
let fetchMock: ReturnType<typeof vi.fn>;

beforeEach(() => {
setupTest();
resetSiteProviderHints();
vi.mocked(isKeylessMode).mockReturnValue(false);
fetchMock = vi.fn(async () => resolveResponse([]));
vi.stubGlobal('fetch', fetchMock);
// Initialize config with test API key
initializeConfig({
apiKey: 'test-api-key',
Expand All @@ -41,9 +53,166 @@ describe('executeScrape', () => {

afterEach(() => {
teardownTest();
resetSiteProviderHints();
vi.unstubAllGlobals();
vi.restoreAllMocks();
vi.clearAllMocks();
});

describe('Alexandria site hint', () => {
it('looks up the page host and prints a hint when a provider covers it', async () => {
fetchMock.mockResolvedValue(resolveResponse([{ id: 'fred' }]));
mockClient.scrape.mockResolvedValue({ markdown: '# Series' });
const stderr = vi
.spyOn(process.stderr, 'write')
.mockImplementation(() => true);

const result = await executeScrape({
url: 'https://fred.stlouisfed.org/series/CPIAUCSL?x=1',
});

expect(result.success).toBe(true);
expect(fetchMock).toHaveBeenCalledTimes(1);
const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit];
expect(url).toBe('https://api.firecrawl.dev/exchange/skills/resolve');
expect(init.method).toBe('POST');
expect(JSON.parse(init.body as string)).toEqual({
urls: ['https://fred.stlouisfed.org/'],
});
expect((init.headers as Record<string, string>).Authorization).toBe(
'Bearer test-api-key'
);
const output = stderr.mock.calls.map(([text]) => String(text)).join('');
expect(output).toContain(
'Alexandria has a structured-data provider for fred.stlouisfed.org (fred).'
);
expect(output).toContain('firecrawl list fred');
expect(output).toContain('firecrawl scrape fred/<capability>');
});

it('prints the hint once per host and looks the host up once', async () => {
fetchMock.mockResolvedValue(resolveResponse([{ id: 'fred' }]));
mockClient.scrape.mockResolvedValue({ markdown: '# Series' });
const stderr = vi
.spyOn(process.stderr, 'write')
.mockImplementation(() => true);

await executeScrape({ url: 'https://fred.stlouisfed.org/a' });
await executeScrape({ url: 'https://fred.stlouisfed.org/b' });

expect(fetchMock).toHaveBeenCalledTimes(1);
const hints = stderr.mock.calls.filter(([text]) =>
String(text).startsWith('Alexandria has')
);
expect(hints).toHaveLength(1);
});

it('prints nothing when no provider matches or the lookup fails', async () => {
mockClient.scrape.mockResolvedValue({ markdown: '# Page' });
const stderr = vi
.spyOn(process.stderr, 'write')
.mockImplementation(() => true);

await executeScrape({ url: 'https://example.com' });
fetchMock.mockResolvedValue(new Response('nope', { status: 503 }));
await executeScrape({ url: 'https://example.org' });
fetchMock.mockRejectedValue(new Error('offline'));
const result = await executeScrape({ url: 'https://example.net' });

expect(result.success).toBe(true);
expect(
stderr.mock.calls.some(([text]) =>
String(text).startsWith('Alexandria has')
)
).toBe(false);
});

it('skips the lookup with --domain-tools, lockdown, keyless mode or a self-hosted API', async () => {
mockClient.scrape.mockResolvedValue({ markdown: '# Page' });

await executeScrape({ url: 'https://example.com', domainTools: true });
await executeScrape({ url: 'https://example.com', lockdown: true });
await executeScrape({
url: 'https://example.com',
apiUrl: 'http://localhost:3002',
});
vi.mocked(isKeylessMode).mockReturnValue(true);
vi.mocked(keylessRequest).mockResolvedValueOnce({
data: { markdown: '# Page' },
});
await executeScrape({ url: 'https://example.com' });

expect(fetchMock).not.toHaveBeenCalled();
});

it('looks up the cloud API given with a trailing slash', async () => {
mockClient.scrape.mockResolvedValue({ markdown: '# Page' });

await executeScrape({
url: 'https://example.com',
apiUrl: 'https://api.firecrawl.dev/',
});

expect(fetchMock).toHaveBeenCalledWith(
'https://api.firecrawl.dev/exchange/skills/resolve',
expect.anything()
);
});

it('keeps the hint for a concurrent same-host scrape when another fails', async () => {
let answer: (response: Response) => void = () => {};
fetchMock.mockImplementation(
() => new Promise<Response>((resolve) => (answer = resolve))
);
let failScrape: (error: Error) => void = () => {};
mockClient.scrape
.mockImplementationOnce(
() => new Promise((_resolve, reject) => (failScrape = reject))
)
.mockResolvedValueOnce({ markdown: '# Series' });
const stderr = vi
.spyOn(process.stderr, 'write')
.mockImplementation(() => true);

const failing = executeScrape({ url: 'https://fred.stlouisfed.org/a' });
const succeeding = executeScrape({
url: 'https://fred.stlouisfed.org/b',
});
failScrape(new Error('boom'));
expect((await failing).success).toBe(false);
answer(resolveResponse([{ id: 'fred' }]));
expect((await succeeding).success).toBe(true);

expect(fetchMock).toHaveBeenCalledTimes(1);
expect(
stderr.mock.calls.some(([text]) =>
String(text).startsWith('Alexandria has')
)
).toBe(true);
});

it('aborts a late lookup instead of waiting for it', async () => {
let signal: AbortSignal | undefined;
fetchMock.mockImplementation(
(_url: string, init: RequestInit) =>
new Promise((_resolve, reject) => {
signal = init.signal as AbortSignal;
signal.addEventListener('abort', () =>
reject(new DOMException('Aborted', 'AbortError'))
);
})
);
mockClient.scrape.mockResolvedValue({ markdown: '# Page' });

const started = Date.now();
const result = await executeScrape({ url: 'https://example.com' });

expect(result.success).toBe(true);
expect(Date.now() - started).toBeLessThan(2_000);
expect(signal?.aborted).toBe(true);
});
});

describe('API call generation', () => {
it('should call scrape with correct URL and default markdown format', async () => {
const mockResponse = { markdown: '# Test Content' };
Expand Down
19 changes: 18 additions & 1 deletion src/commands/scrape.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@ import { executeMap } from './map';
import { getStatus } from './status';
import { requireAlexandriaKey } from './alexandria';
import { withoutToolPrice } from '../utils/alexandria-prices';
import {
lookupSiteProviders,
releaseSiteProviderLookup,
settleSiteProviderHint,
} from '../utils/alexandria-hint';

const HINT_WAIT_MS = 500;

/**
* Output timing information if requested
Expand Down Expand Up @@ -146,6 +153,11 @@ export async function executeScrape(

// Execute scrape with timing - only wrap the scrape call in try-catch
const requestStartTime = Date.now();
const keyless = isKeylessMode(options.apiKey, options.apiUrl);
const siteLookup =
options.domainTools || options.lockdown || keyless
? undefined
: lookupSiteProviders(options.url, options);

try {
if (options.toolDetail !== undefined)
Expand All @@ -155,7 +167,7 @@ export async function executeScrape(
scrapeParams.domainTools = true;
}
let result: any;
if (isKeylessMode(options.apiKey, options.apiUrl)) {
if (keyless) {
// Keyless free tier: header-less request. The API identifies the CLI via
// the `integration: 'cli'` field already in scrapeParams.
const json = await keylessRequest('/v2/scrape', {
Expand Down Expand Up @@ -190,13 +202,18 @@ export async function executeScrape(
}
}

const hint =
siteLookup && (await settleSiteProviderHint(siteLookup, HINT_WAIT_MS));
if (hint) process.stderr.write(`${hint}\n`);

return {
success: true,
data: Array.isArray(result?.tools)
? { ...result, tools: result.tools.map(withoutToolPrice) }
: result,
};
} catch (error) {
if (siteLookup) releaseSiteProviderLookup(siteLookup);
const requestEndTime = Date.now();
outputTiming(options, requestStartTime, requestEndTime, error);

Expand Down
Loading
Loading