From 084103c1e4dc9b4791587f31e0ebc56430d14cd3 Mon Sep 17 00:00:00 2001 From: mogery Date: Wed, 11 Nov 2026 19:09:01 +0000 Subject: [PATCH] chore: remove the hidden create and sql commands Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- src/__tests__/cli-aliases.test.ts | 27 -- src/__tests__/commands/sql.test.ts | 85 ---- src/commands/create.ts | 99 ---- src/commands/sql.ts | 57 --- src/index.ts | 9 - src/utils/agent-scaffold/create-flow.ts | 600 ------------------------ src/utils/agent-scaffold/credentials.ts | 63 --- src/utils/agent-scaffold/manifest.ts | 207 -------- src/utils/agent-scaffold/scaffold.ts | 246 ---------- src/utils/agent-scaffold/ui.ts | 45 -- 10 files changed, 1438 deletions(-) delete mode 100644 src/__tests__/commands/sql.test.ts delete mode 100644 src/commands/create.ts delete mode 100644 src/commands/sql.ts delete mode 100644 src/utils/agent-scaffold/create-flow.ts delete mode 100644 src/utils/agent-scaffold/credentials.ts delete mode 100644 src/utils/agent-scaffold/manifest.ts delete mode 100644 src/utils/agent-scaffold/scaffold.ts delete mode 100644 src/utils/agent-scaffold/ui.ts diff --git a/src/__tests__/cli-aliases.test.ts b/src/__tests__/cli-aliases.test.ts index d6ac5eba3c..977e002bca 100644 --- a/src/__tests__/cli-aliases.test.ts +++ b/src/__tests__/cli-aliases.test.ts @@ -39,33 +39,6 @@ describe('CLI compatibility aliases', { timeout: 30000 }, () => { }); } - testWithBuiltCli( - 'sql preserves experimental aliases and execution options', - () => { - const flags = ['SHOW CATEGORIES', '--execute', '--pretty']; - const canonical = run(['sql', ...flags]); - expect(canonical.status, canonical.stderr).toBe(0); - expect(JSON.parse(canonical.stdout)).toMatchObject({ - calls: [ - { - provider: 'firecrawl', - capability: 'sql', - options: { query: 'SHOW CATEGORIES', execute: true }, - }, - ], - options: { pretty: true }, - }); - for (const prefix of [ - ['x', 'sql'], - ['experimental', 'sql'], - ]) { - const alias = run([...prefix, ...flags]); - expect(alias.status, alias.stderr).toBe(0); - expect(alias.stdout).toBe(canonical.stdout); - } - } - ); - testWithBuiltCli( 'credits preserves credit-usage options and authentication', () => { diff --git a/src/__tests__/commands/sql.test.ts b/src/__tests__/commands/sql.test.ts deleted file mode 100644 index ab5cf6d195..0000000000 --- a/src/__tests__/commands/sql.test.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { afterEach, expect, it, vi } from 'vitest'; -import { Readable } from 'node:stream'; -import { mkdtemp, writeFile, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { createSqlCommand, readSqlInput } from '../../commands/sql'; -import { handleAlexandria } from '../../commands/alexandria'; - -vi.mock('../../commands/alexandria', () => ({ handleAlexandria: vi.fn() })); -afterEach(() => vi.clearAllMocks()); - -it.each([false, true])('forwards raw SQL with execute=%s', async (execute) => { - const query = 'SELECT * FROM "example/tool" WHERE name = \'Nike\' LIMIT 1'; - await createSqlCommand().parseAsync( - [ - query, - ...(execute ? ['--execute'] : []), - '--pretty', - '--request-id', - 'request', - ], - { from: 'user' } - ); - expect(handleAlexandria).toHaveBeenCalledWith( - [{ provider: 'firecrawl', capability: 'sql', options: { query, execute } }], - expect.objectContaining({ pretty: true, requestId: 'request' }) - ); -}); - -it('reads multiline stdin without changing quotes', async () => { - const query = - 'WITH a AS (\n SELECT * FROM "a/b" LIMIT 1\n) SELECT * FROM a LIMIT 1'; - expect(await readSqlInput(undefined, undefined, Readable.from([query]))).toBe( - query - ); -}); - -it('reads a file and forwards it to the existing handler', async () => { - const dir = await mkdtemp(join(tmpdir(), 'sql-cli-')); - try { - const path = join(dir, 'query.sql'); - await writeFile(path, 'SHOW TABLES LIMIT 1'); - await createSqlCommand().parseAsync(['--file', path], { from: 'user' }); - expect(handleAlexandria).toHaveBeenCalledWith( - [ - { - provider: 'firecrawl', - capability: 'sql', - options: { query: 'SHOW TABLES LIMIT 1', execute: false }, - }, - ], - expect.anything() - ); - } finally { - await rm(dir, { recursive: true, force: true }); - } -}); - -it('rejects ambiguous sources without dispatch', async () => { - await expect( - createSqlCommand().parseAsync(['HELP', '--file', 'query.sql'], { - from: 'user', - }) - ).rejects.toThrow('either'); - expect(handleAlexandria).not.toHaveBeenCalled(); -}); - -it('rejects missing, empty and oversized input', async () => { - await expect( - readSqlInput( - undefined, - undefined, - Object.assign(Readable.from([]), { isTTY: true }) - ) - ).rejects.toThrow('Provide'); - await expect( - readSqlInput(undefined, undefined, Readable.from([' '])) - ).rejects.toThrow('empty'); - await expect(readSqlInput('x'.repeat(16001), undefined)).rejects.toThrow( - '16,000' - ); - await expect( - readSqlInput(undefined, undefined, Readable.from(['x'.repeat(16001)])) - ).rejects.toThrow('16,000'); -}); diff --git a/src/commands/create.ts b/src/commands/create.ts deleted file mode 100644 index c54f3d6e77..0000000000 --- a/src/commands/create.ts +++ /dev/null @@ -1,99 +0,0 @@ -/** - * `firecrawl create` command — scaffolds Firecrawl starter projects. - * - * Hidden from --help until the flow is battle-tested in the wild. The - * scaffolder is vendored under `src/utils/agent-scaffold/` (mirrored from - * `firecrawl/web-agent`). At runtime it clones the public agent repo - * to get templates — no separate npm package for the agent CLI. - * - * Once visible, the command tree will grow to include additional kinds - * (scrape, browser, ai, app). For now, `agent` is the only kind. - */ - -import { Command } from 'commander'; -import type { CreateOptions } from '../utils/agent-scaffold/create-flow'; - -function collect(val: string, acc: string[]): string[] { - acc.push(val); - return acc; -} - -/** - * Build the `agent` subcommand. Flag surface mirrors the upstream agent CLI. - * The scaffold flow itself lives in `utils/agent-scaffold/create-flow.ts`. - */ -function createAgentSubcommand(): Command { - return new Command('agent') - .description( - 'Scaffold a Firecrawl Agent project (defaults to the Next.js template)' - ) - .argument('[project-name]', 'Project directory name') - .option( - '-t, --template ', - 'Template variant (next, express, library)', - 'next' - ) - .option( - '--provider ', - 'Orchestrator model provider (anthropic, openai, google, gateway, custom-openai)' - ) - .option('--model ', 'Orchestrator model ID') - .option( - '--sub-agent-provider ', - 'Sub-agent model provider (defaults to orchestrator)' - ) - .option( - '--sub-agent-model ', - 'Sub-agent model ID (defaults to orchestrator)' - ) - .option( - '--from ', - 'External repo (user/repo) or local path with agent-manifest.json' - ) - .option('--api-key ', 'Firecrawl API key') - .option( - '--key ', - 'Provider API key (repeatable, e.g. --key anthropic=sk-...)', - collect, - [] - ) - .option('--skip-install', 'Skip npm install') - .action( - async ( - projectName: string | undefined, - options: CreateOptions & Record - ) => { - // Lazy-load the scaffolder so startup cost (inquirer, git) only hits - // users who actually invoke `create`. Keeps the rest of the CLI snappy. - const { handleCreate } = - await import('../utils/agent-scaffold/create-flow'); - await handleCreate(projectName, { - template: options.template, - provider: options.provider, - model: options.model, - subAgentProvider: options.subAgentProvider, - subAgentModel: options.subAgentModel, - from: options.from, - apiKey: options.apiKey, - key: Array.isArray(options.key) - ? (options.key as string[]) - : undefined, - skipInstall: options.skipInstall as boolean | undefined, - }); - } - ); -} - -/** - * Top-level `firecrawl create` command. For now it only wires the `agent` - * subcommand; future kinds (scrape, browser, ai, app) register here. - */ -export function createCreateCommand(): Command { - const cmd = new Command('create').description( - 'Scaffold a Firecrawl starter project' - ); - - cmd.addCommand(createAgentSubcommand()); - - return cmd; -} diff --git a/src/commands/sql.ts b/src/commands/sql.ts deleted file mode 100644 index a2fedadab9..0000000000 --- a/src/commands/sql.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { Command } from 'commander'; -import { readFile } from 'node:fs/promises'; -import { handleAlexandria } from './alexandria'; - -export async function readSqlInput( - query: string | undefined, - file: string | undefined, - stdin: AsyncIterable & { isTTY?: boolean } = process.stdin -): Promise { - if (query !== undefined && file !== undefined) - throw new Error('Use either a query or --file.'); - if (file !== undefined) query = await readFile(file, 'utf8'); - if (query === undefined) { - if (stdin.isTTY) throw new Error('Provide a query, --file, or stdin.'); - query = ''; - for await (const chunk of stdin) { - query += chunk.toString(); - if (query.length > 16000) - throw new Error('SQL must be at most 16,000 characters.'); - } - } - if (!query.trim()) throw new Error('SQL must not be empty.'); - if (query.length > 16000) - throw new Error('SQL must be at most 16,000 characters.'); - return query; -} - -export function createSqlCommand(): Command { - return new Command('sql') - .description('Experimental Alexandria syntax') - .argument('[query]', 'SQL statement') - .option('-f, --file ', 'Read SQL from a file') - .option( - '--execute', - 'Execute paid provider calls; defaults to preview', - false - ) - .option('-k, --api-key ', 'Firecrawl API key') - .option('--api-url ', 'Firecrawl API URL') - .option('--request-id ', 'Request ID') - .option('-o, --output ', 'Output file') - .option('--json', 'Output JSON') - .option('--pretty', 'Format JSON') - .action(async (query, options) => { - const statement = await readSqlInput(query, options.file); - await handleAlexandria( - [ - { - provider: 'firecrawl', - capability: 'sql', - options: { query: statement, execute: options.execute }, - }, - ], - options - ); - }); -} diff --git a/src/index.ts b/src/index.ts index 64fd32b03a..b160aa0d43 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,7 +6,6 @@ */ import { Command, InvalidArgumentError, Option } from 'commander'; -import { createSqlCommand } from './commands/sql'; import { addFormatsAlias } from './utils/format-option'; import { addAlexandriaScrapeOptions, @@ -104,7 +103,6 @@ import type { SearchSource, SearchCategory } from './types/search'; import type { ScrapeFormat } from './types/scrape'; import type { RelatedPapersOptions } from './types/research'; import type { AgentExchangeOptions, AgentWebhookConfig } from 'firecrawl'; -import { createCreateCommand } from './commands/create'; import { createListCommand, createAlexandriaCommand } from './commands/list'; // Initialize global configuration from environment variables @@ -2688,11 +2686,6 @@ program.addCommand(createInteractCommand()); // Hidden: deprecated browser command (still works, just not in --help) program.addCommand(createBrowserCommand(), { hidden: true }); -// Hidden: `firecrawl create ` — scaffolds Firecrawl starter projects. -// Undocumented until `firecrawl-agent-cli` is published to npm; flip to -// visible by removing `{ hidden: true }`. -program.addCommand(createCreateCommand(), { hidden: true }); - // Experimental: download command const experimental = new Command('experimental') .description('Experimental commands (download)') @@ -2704,8 +2697,6 @@ Shorthand: "firecrawl x" is an alias for "firecrawl experimental". ` ); experimental.addCommand(createDownloadCommand()); -program.addCommand(createSqlCommand(), { hidden: true }); -experimental.addCommand(createSqlCommand(), { hidden: true }); program.addCommand(experimental); program diff --git a/src/utils/agent-scaffold/create-flow.ts b/src/utils/agent-scaffold/create-flow.ts deleted file mode 100644 index 582742cb8f..0000000000 --- a/src/utils/agent-scaffold/create-flow.ts +++ /dev/null @@ -1,600 +0,0 @@ -/** - * Vendored from firecrawl/web-agent:.internal/cli/src/commands/init.ts - * - * This is the interactive-to-non-interactive create flow. The original file - * also exposed a `createInitCommand` commander builder — we drop that here - * because the root CLI's `src/commands/create.ts` owns the command surface. - * `handleCreate` (renamed from `handleInit`) is what we export. - * - * Keep in sync with upstream when the create flow evolves. - */ - -import { select, password, input } from '@inquirer/prompts'; -import * as path from 'path'; -import { spawn } from 'child_process'; -import { - getTemplates, - getProviders, - getTemplate, - loadExternalManifest, -} from './manifest'; -import type { ProviderEntry, TemplateEntry } from './manifest'; -import { resolveFirecrawlApiKey } from './credentials'; -import { scaffoldProject } from './scaffold'; -import { - printBanner, - success, - warn, - info, - dim, - reset, - green, - bold, -} from './ui'; - -function resolveSpawnCommand(command: string): string { - if (process.platform === 'win32' && command === 'npm') { - return 'npm.cmd'; - } - return command; -} - -export interface CreateOptions { - template?: string; - provider?: string; - model?: string; - subAgentProvider?: string; - subAgentModel?: string; - from?: string; - apiKey?: string; - key?: string[]; - skipInstall?: boolean; -} - -function getSelectedProvider( - providers: ProviderEntry[], - providerId?: string -): ProviderEntry | undefined { - return providers.find((provider) => provider.id === providerId); -} - -function parseKeyFlags(keys: string[]): Record { - const map: Record = {}; - const providerEnvMap: Record = { - anthropic: 'ANTHROPIC_API_KEY', - openai: 'OPENAI_API_KEY', - google: 'GOOGLE_GENERATIVE_AI_API_KEY', - gateway: 'AI_GATEWAY_API_KEY', - }; - - for (const entry of keys) { - const eq = entry.indexOf('='); - if (eq === -1) continue; - const provider = entry.slice(0, eq).toLowerCase(); - const value = entry.slice(eq + 1); - const envVar = providerEnvMap[provider] ?? provider.toUpperCase(); - map[envVar] = value; - } - return map; -} - -/** - * Prompt the user to pick a provider+model pair from the available manifest. - * Used by both the initial orchestrator prompt and the review-loop "Change…" - * actions so changes stay in sync. - */ -async function promptProviderAndModel( - availableProviders: ProviderEntry[], - message: string -): Promise<{ provider: ProviderEntry; modelId: string }> { - const providerId = await select({ - message, - choices: availableProviders - .filter((p) => p.models && p.models.length > 0) - .map((provider) => ({ - name: `${provider.name} ${dim}${provider.models[0].name}${reset}`, - value: provider.id, - })), - }); - const provider = getSelectedProvider(availableProviders, providerId)!; - - let modelId: string; - if (provider.id === 'custom-openai') { - modelId = - (await input({ message: 'Model ID', default: 'gpt-4o' })).trim() || - 'gpt-4o'; - } else if (provider.models.length > 1) { - modelId = await select({ - message: 'Model', - choices: provider.models.map((m) => ({ name: m.name, value: m.id })), - }); - } else { - modelId = provider.models[0]?.id ?? 'gpt-4o'; - } - - return { provider, modelId }; -} - -export async function handleCreate( - rawName: string | undefined, - options: CreateOptions -): Promise { - printBanner(); - - // --- Project name: arg > interactive prompt > default --- - let projectName: string; - - if (rawName) { - projectName = rawName; - } else if (process.stdin.isTTY) { - projectName = - ( - await input({ - message: 'Project name', - default: 'my-firecrawl-agent', - }) - ).trim() || 'my-firecrawl-agent'; - } else { - projectName = 'my-firecrawl-agent'; - } - - // Load external manifest if --from is provided - if (options.from) { - try { - await loadExternalManifest(options.from); - success(`Loaded manifest from ${options.from}`); - } catch (err) { - warn(err instanceof Error ? err.message : String(err)); - process.exit(1); - } - console.log(''); - } - - const availableProviders = getProviders(); - - // --- Template selection (only interactive prompt if not passed via flag) --- - const templates = getTemplates(); - let template: TemplateEntry; - - if (options.template) { - const found = getTemplate(options.template); - if (!found) { - warn( - `Unknown template "${options.template}". Available: ${templates.map((t) => t.id).join(', ')}` - ); - process.exit(1); - } - template = found; - } else { - const templateId = await select({ - message: 'Template', - choices: templates.map((t) => ({ - name: `${t.name} ${dim}${t.description}${reset}`, - value: t.id, - })), - }); - template = getTemplate(templateId)!; - } - - let selectedProvider = getSelectedProvider( - availableProviders, - options.provider - ); - if (options.provider && !selectedProvider) { - warn( - `Unknown provider "${options.provider}". Available: ${availableProviders.map((p) => p.id).join(', ')}` - ); - process.exit(1); - } - - if (!selectedProvider) { - if (process.stdin.isTTY) { - const providerId = await select({ - message: 'Default model provider', - choices: availableProviders - .filter((p) => p.models && p.models.length > 0) - .map((provider) => ({ - name: `${provider.name} ${dim}${provider.models[0].name}${reset}`, - value: provider.id, - })), - }); - selectedProvider = getSelectedProvider(availableProviders, providerId)!; - } else { - selectedProvider = - getSelectedProvider(availableProviders, 'google') ?? - availableProviders[0]; - } - } - - // --- Orchestrator model selection --- - let selectedModelId: string; - - if (options.model) { - selectedModelId = options.model; - } else if (selectedProvider.id === 'custom-openai') { - selectedModelId = ( - await input({ - message: 'Model ID', - default: 'gpt-4o', - }) - ).trim(); - } else if (selectedProvider.models.length > 1 && process.stdin.isTTY) { - selectedModelId = await select({ - message: 'Default model', - choices: selectedProvider.models.map((m) => ({ - name: m.name, - value: m.id, - })), - }); - } else { - selectedModelId = selectedProvider.models[0]?.id ?? 'gpt-4o'; - } - - // --- Sub-agent model selection --- - let selectedSubProvider: ProviderEntry = selectedProvider; - let selectedSubModelId: string = selectedModelId; - - if (options.subAgentProvider || options.subAgentModel) { - const subProv = options.subAgentProvider - ? getSelectedProvider(availableProviders, options.subAgentProvider) - : selectedProvider; - if (options.subAgentProvider && !subProv) { - warn( - `Unknown sub-agent provider "${options.subAgentProvider}". Available: ${availableProviders.map((p) => p.id).join(', ')}` - ); - process.exit(1); - } - selectedSubProvider = subProv ?? selectedProvider; - selectedSubModelId = - options.subAgentModel ?? - (selectedSubProvider.id === selectedProvider.id - ? selectedModelId - : (selectedSubProvider.models[0]?.id ?? selectedModelId)); - } else if (process.stdin.isTTY && !options.model) { - const subChoice = await select({ - message: 'Sub-agent model', - choices: [ - { - name: `Same as orchestrator ${dim}(${selectedModelId})${reset}`, - value: '__same__', - }, - ...availableProviders.flatMap((p) => - p.models.map((m) => ({ - name: `${p.name} — ${m.name}`, - value: `${p.id}::${m.id}`, - })) - ), - ], - }); - if (subChoice !== '__same__') { - const [pId, mId] = subChoice.split('::'); - const p = getSelectedProvider(availableProviders, pId); - if (p) { - selectedSubProvider = p; - selectedSubModelId = mId; - } - } - } - - // --- Custom OpenAI endpoint --- - let customEndpoint: string | undefined; - if (selectedProvider.endpointEnvVar && process.stdin.isTTY) { - customEndpoint = - ( - await input({ - message: `Base URL ${dim}(OpenAI-compatible endpoint)${reset}`, - default: 'https://api.openai.com/v1', - }) - ).trim() || undefined; - } - - // --- Collect all env vars silently --- - const envVars: Record = {}; - const missing = new Set(); - - envVars.MODEL_PROVIDER = selectedProvider.id; - envVars.MODEL_ID = selectedModelId; - - if (selectedProvider.endpointEnvVar && customEndpoint) { - envVars[selectedProvider.endpointEnvVar] = customEndpoint; - } - - // Firecrawl API key: flag > env > credentials > prompt - if (options.apiKey) { - envVars.FIRECRAWL_API_KEY = options.apiKey; - } else { - const resolved = await resolveFirecrawlApiKey(); - if (resolved) { - envVars.FIRECRAWL_API_KEY = resolved.key; - } - } - - // Provider keys from --key flags - const flagKeys = parseKeyFlags(options.key ?? []); - Object.assign(envVars, flagKeys); - - // Auto-detect remaining provider keys from environment - for (const provider of availableProviders) { - if (!envVars[provider.envVar] && process.env[provider.envVar]) { - envVars[provider.envVar] = process.env[provider.envVar]!; - } - } - - // Prompt for missing required keys when running interactively - if (!envVars.FIRECRAWL_API_KEY) { - const key = await password({ - message: `Firecrawl API key ${dim}(https://firecrawl.dev/app/api-keys)${reset}`, - }); - if (key) { - envVars.FIRECRAWL_API_KEY = key; - } else { - missing.add('FIRECRAWL_API_KEY'); - } - } - - if (!envVars[selectedProvider.envVar] && process.stdin.isTTY) { - const key = await password({ - message: `${selectedProvider.name} API key ${dim}(${selectedProvider.hint})${reset}`, - }); - if (key) { - envVars[selectedProvider.envVar] = key; - } else { - missing.add(selectedProvider.envVar); - } - } - - if ( - selectedSubProvider.id !== selectedProvider.id && - !envVars[selectedSubProvider.envVar] && - process.stdin.isTTY - ) { - const key = await password({ - message: `${selectedSubProvider.name} API key ${dim}(${selectedSubProvider.hint})${reset}`, - }); - if (key) { - envVars[selectedSubProvider.envVar] = key; - } else { - missing.add(selectedSubProvider.envVar); - } - } - - for (const envVar of template.optionalEnvVars) { - if (!envVars[envVar]) missing.add(envVar); - } - - // --- Config summary review loop --- - const fullyFlagged = !!( - options.template && - (options.apiKey || envVars.FIRECRAWL_API_KEY) && - options.provider && - options.model - ); - if (process.stdin.isTTY && !fullyFlagged) { - let confirmed = false; - while (!confirmed) { - printConfigSummary({ - template, - orchProvider: selectedProvider, - orchModelId: selectedModelId, - subProvider: selectedSubProvider, - subModelId: selectedSubModelId, - envVars, - missing, - }); - const action = await select({ - message: 'Review', - choices: [ - { name: 'Continue', value: 'continue' }, - { name: 'Change orchestrator model', value: 'orch' }, - { name: 'Change sub-agent model', value: 'sub' }, - { name: 'Cancel', value: 'cancel' }, - ], - }); - if (action === 'cancel') { - info('Cancelled.'); - process.exit(0); - } - if (action === 'orch') { - const picked = await promptProviderAndModel( - availableProviders, - 'Orchestrator model' - ); - const prev = selectedProvider; - selectedProvider = picked.provider; - selectedModelId = picked.modelId; - envVars.MODEL_PROVIDER = selectedProvider.id; - envVars.MODEL_ID = selectedModelId; - if (prev.envVar !== selectedProvider.envVar) { - if (!envVars[selectedProvider.envVar]) { - const key = await password({ - message: `${selectedProvider.name} API key ${dim}(${selectedProvider.hint})${reset}`, - }); - if (key) envVars[selectedProvider.envVar] = key; - else missing.add(selectedProvider.envVar); - } - } - continue; - } - if (action === 'sub') { - const sameChoice = await select({ - message: 'Sub-agent model', - choices: [ - { - name: `Same as orchestrator ${dim}(${selectedModelId})${reset}`, - value: '__same__', - }, - { name: 'Different provider / model', value: '__different__' }, - ], - }); - if (sameChoice === '__same__') { - selectedSubProvider = selectedProvider; - selectedSubModelId = selectedModelId; - } else { - const picked = await promptProviderAndModel( - availableProviders, - 'Sub-agent model' - ); - selectedSubProvider = picked.provider; - selectedSubModelId = picked.modelId; - if ( - selectedSubProvider.id !== selectedProvider.id && - !envVars[selectedSubProvider.envVar] - ) { - const key = await password({ - message: `${selectedSubProvider.name} API key ${dim}(${selectedSubProvider.hint})${reset}`, - }); - if (key) envVars[selectedSubProvider.envVar] = key; - else missing.add(selectedSubProvider.envVar); - } - } - continue; - } - confirmed = true; - } - } - - // --- Scaffold --- - const projectDir = path.resolve(process.cwd(), projectName); - console.log(''); - info(`Creating a new Firecrawl Agent app in ${projectDir}`); - console.log(''); - - await scaffoldProject({ - projectDir, - template, - envVars, - selectedProvider: selectedProvider.id, - defaultModelId: envVars.MODEL_ID, - subAgentProvider: selectedSubProvider.id, - subAgentModelId: selectedSubModelId, - skipInstall: options.skipInstall, - }); - - // --- Summary --- - console.log(''); - console.log(` ${green}${bold}Ready!${reset} ${projectDir}`); - console.log(''); - - const detected = Object.keys(envVars).filter( - (k) => /_API_KEY$/.test(k) && envVars[k] - ); - if (detected.length > 0) { - info(`Keys: ${detected.join(', ')}`); - } - info(`Orchestrator: ${selectedProvider.name} (${envVars.MODEL_ID})`); - if ( - selectedSubProvider.id === selectedProvider.id && - selectedSubModelId === selectedModelId - ) { - info(`Sub-agent: same as orchestrator`); - } else { - info(`Sub-agent: ${selectedSubProvider.name} (${selectedSubModelId})`); - } - if (missing.size > 0) { - info( - `Missing: ${Array.from(missing).join(', ')} ${dim}(add to .env later)${reset}` - ); - } - console.log(''); - - // --- What next? --- - if (fullyFlagged || !process.stdin.isTTY) { - console.log(` cd ${projectName} && ${template.devCommand}`); - console.log(''); - return; - } - - const action = await select({ - message: 'Next', - choices: [ - { - name: `Start dev server ${dim}${template.devCommand}${reset}`, - value: 'dev', - }, - { name: 'Exit', value: 'exit' }, - ], - }); - - if (action === 'dev') { - console.log(''); - const [cmd, ...args] = template.devCommand.split(' '); - spawn(resolveSpawnCommand(cmd), args, { - cwd: projectDir, - stdio: 'inherit', - }); - } else { - console.log(` cd ${projectName} && ${template.devCommand}`); - console.log(''); - } -} - -function printConfigSummary(args: { - template: TemplateEntry; - orchProvider: ProviderEntry; - orchModelId: string; - subProvider: ProviderEntry; - subModelId: string; - envVars: Record; - missing: Set; -}): void { - const { - template, - orchProvider, - orchModelId, - subProvider, - subModelId, - envVars, - missing, - } = args; - const sameAsOrch = - subProvider.id === orchProvider.id && subModelId === orchModelId; - const mask = (v?: string) => (v ? v.slice(0, 5) + '••••' + v.slice(-4) : ''); - console.log(''); - console.log(` ${bold}Config${reset}`); - console.log(` ${dim}────────────────────────────────${reset}`); - console.log( - ` Orchestrator ${green}${orchProvider.name}${reset} ${dim}·${reset} ${orchModelId}` - ); - console.log( - ` Sub-agent ${green}${subProvider.name}${reset} ${dim}·${reset} ${subModelId}` + - (sameAsOrch ? ` ${dim}(same as orchestrator)${reset}` : '') - ); - console.log(` Template ${template.name}`); - const fcKey = envVars.FIRECRAWL_API_KEY; - console.log( - ` Firecrawl key ${fcKey ? `${mask(fcKey)} ${green}✓${reset}` : `${dim}(missing)${reset} !`}` - ); - const orchKey = envVars[orchProvider.envVar]; - console.log( - ` ${orchProvider.name} key` + - ' '.repeat(Math.max(1, 14 - orchProvider.name.length - 4)) + - (orchKey - ? `${mask(orchKey)} ${green}✓${reset}` - : `${dim}(missing)${reset} !`) - ); - if (!sameAsOrch) { - const subKey = envVars[subProvider.envVar]; - console.log( - ` ${subProvider.name} key (sub)` + - ' '.repeat(Math.max(1, 8 - subProvider.name.length)) + - (subKey - ? `${mask(subKey)} ${green}✓${reset}` - : `${dim}(missing)${reset} !`) - ); - } - if (missing.size > 0) { - console.log( - ` ${dim}Other missing: ${ - Array.from(missing) - .filter( - (m) => - m !== orchProvider.envVar && - m !== subProvider.envVar && - m !== 'FIRECRAWL_API_KEY' - ) - .join(', ') || '—' - }${reset}` - ); - } - console.log(''); -} diff --git a/src/utils/agent-scaffold/credentials.ts b/src/utils/agent-scaffold/credentials.ts deleted file mode 100644 index 091ba6ce08..0000000000 --- a/src/utils/agent-scaffold/credentials.ts +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Vendored from firecrawl/web-agent:.internal/cli/src/utils/credentials.ts - * - * Resolves the Firecrawl API key from env or the shared `firecrawl-cli` config - * directory. Same path as the root CLI's own credentials store (that's the - * point — both CLIs read from the same place), so keeping this as a light - * standalone copy avoids coupling the scaffolder to the root CLI's richer - * credentials module. - */ - -import * as fs from 'fs'; -import * as path from 'path'; -import * as os from 'os'; - -interface StoredCredentials { - apiKey?: string; - apiUrl?: string; -} - -function getConfigDir(): string { - const homeDir = os.homedir(); - switch (os.platform()) { - case 'darwin': - return path.join( - homeDir, - 'Library', - 'Application Support', - 'firecrawl-cli' - ); - case 'win32': - return path.join(homeDir, 'AppData', 'Roaming', 'firecrawl-cli'); - default: - return path.join(homeDir, '.config', 'firecrawl-cli'); - } -} - -export function loadFirecrawlCredentials(): StoredCredentials | null { - try { - const credPath = path.join(getConfigDir(), 'credentials.json'); - if (!fs.existsSync(credPath)) return null; - return JSON.parse(fs.readFileSync(credPath, 'utf-8')) as StoredCredentials; - } catch { - return null; - } -} - -export async function resolveFirecrawlApiKey(): Promise<{ - key: string; - source: 'env' | 'credentials' | 'prompt'; -} | null> { - // 1. Environment variable - if (process.env.FIRECRAWL_API_KEY) { - return { key: process.env.FIRECRAWL_API_KEY, source: 'env' }; - } - - // 2. Stored credentials from firecrawl-cli - const creds = loadFirecrawlCredentials(); - if (creds?.apiKey) { - return { key: creds.apiKey, source: 'credentials' }; - } - - return null; -} diff --git a/src/utils/agent-scaffold/manifest.ts b/src/utils/agent-scaffold/manifest.ts deleted file mode 100644 index e36951d6c6..0000000000 --- a/src/utils/agent-scaffold/manifest.ts +++ /dev/null @@ -1,207 +0,0 @@ -/** - * Vendored from firecrawl/web-agent:.internal/cli/src/utils/manifest.ts - * - * Divergence from upstream: removed the "look for agent-manifest.json next to - * the bundled CLI" branch — when this code runs inside the root firecrawl-cli - * npm package there is no bundled manifest. `loadManifest()` always clones - * the public agent repo into a tmp dir and reads the manifest from there, - * which is exactly what `loadExternalManifestSync` already does. - * - * Keep in sync with upstream when the manifest schema evolves. - */ - -import * as path from 'path'; -import * as fs from 'fs'; -import { execSync } from 'child_process'; -import { info } from './ui'; - -export interface TemplateEntry { - id: string; - name: string; - description: string; - path: string; - requiredEnvVars: string[]; - optionalEnvVars: string[]; - envFile?: string; - devCommand: string; - deploy: string[]; -} - -export interface ModelEntry { - id: string; - name: string; -} - -export interface ProviderEntry { - id: string; - name: string; - envVar: string; - hint: string; - models: ModelEntry[]; - endpointEnvVar?: string; -} - -export interface Manifest { - version: number; - templates: TemplateEntry[]; - providers: ProviderEntry[]; -} - -let cached: Manifest | null = null; -let cachedSourceRoot: string | null = null; - -const DEFAULT_REMOTE = 'firecrawl/web-agent'; - -export function loadManifest(): Manifest { - if (cached) return cached; - const { manifest } = loadExternalManifestSync(DEFAULT_REMOTE); - return manifest; -} - -function cloneRepo(source: string, dest: string): void { - const ghToken = process.env.GITHUB_TOKEN || process.env.GH_TOKEN; - - // Try gh CLI first (handles private repos via gh auth) - try { - execSync(`gh repo clone ${source} "${dest}" -- --depth 1`, { - stdio: 'pipe', - }); - return; - } catch {} - - // Fall back to git clone with token if available - const cloneUrl = ghToken - ? `https://${ghToken}@github.com/${source}.git` - : `https://github.com/${source}.git`; - execSync(`git clone --depth 1 ${cloneUrl} "${dest}"`, { stdio: 'pipe' }); -} - -function loadExternalManifestSync(source: string): { - manifest: Manifest; - sourceRoot: string; -} { - const tmpDir = fs.mkdtempSync(path.join(require('os').tmpdir(), 'fc-agent-')); - try { - cloneRepo(source, tmpDir); - } catch { - // Default remote is public once launched; until then, gh auth covers - // insiders and everyone else sees this hint. - throw new Error( - `Could not clone ${source}. If the repo isn't public yet, authenticate with \`gh auth login\` or set GITHUB_TOKEN. Otherwise, check your network / git install.` - ); - } - const manifestPath = path.join(tmpDir, 'agent-manifest.json'); - if (!fs.existsSync(manifestPath)) { - // Upstream currently ships the manifest at `.internal/cli/agent-manifest.json`. - // Fall back to that location so the scaffolder keeps working until the - // manifest is promoted to repo root. - const nestedPath = path.join( - tmpDir, - '.internal', - 'cli', - 'agent-manifest.json' - ); - if (fs.existsSync(nestedPath)) { - const manifest = JSON.parse( - fs.readFileSync(nestedPath, 'utf-8') - ) as Manifest; - cached = manifest; - cachedSourceRoot = tmpDir; - return { manifest, sourceRoot: tmpDir }; - } - throw new Error(`No agent-manifest.json found in ${source}`); - } - const manifest = JSON.parse( - fs.readFileSync(manifestPath, 'utf-8') - ) as Manifest; - cached = manifest; - cachedSourceRoot = tmpDir; - return { manifest, sourceRoot: tmpDir }; -} - -/** - * Load manifest from an external GitHub repo or local path. - * Returns the manifest and the root directory where templates live. - * - * Supported sources: - * - "user/repo" — clones from GitHub, reads agent-manifest.json - * - "/absolute/path" or "./relative" — reads from local directory - */ -export async function loadExternalManifest(source: string): Promise<{ - manifest: Manifest; - sourceRoot: string; -}> { - // Local path - if (source.startsWith('/') || source.startsWith('.')) { - const absPath = path.resolve(source); - const manifestPath = path.join(absPath, 'agent-manifest.json'); - if (fs.existsSync(manifestPath)) { - const manifest = JSON.parse( - fs.readFileSync(manifestPath, 'utf-8') - ) as Manifest; - cached = manifest; - cachedSourceRoot = absPath; - return { manifest, sourceRoot: absPath }; - } - // Same upstream fallback as the remote clone path: while the manifest - // lives under `.internal/cli/` rather than repo root, still accept - // local agent-repo checkouts. Source root stays at repo root so - // templates resolve correctly. - const nestedPath = path.join( - absPath, - '.internal', - 'cli', - 'agent-manifest.json' - ); - if (fs.existsSync(nestedPath)) { - const manifest = JSON.parse( - fs.readFileSync(nestedPath, 'utf-8') - ) as Manifest; - cached = manifest; - cachedSourceRoot = absPath; - return { manifest, sourceRoot: absPath }; - } - throw new Error(`No agent-manifest.json found in ${absPath}`); - } - - // GitHub repo (user/repo format) - const tmpDir = fs.mkdtempSync(path.join(require('os').tmpdir(), 'fc-agent-')); - info(`Cloning ${source}...`); - try { - cloneRepo(source, tmpDir); - } catch { - throw new Error( - `Failed to clone ${source} - for private repos, install the gh CLI (gh auth login) or set GITHUB_TOKEN` - ); - } - - const manifestPath = path.join(tmpDir, 'agent-manifest.json'); - if (!fs.existsSync(manifestPath)) { - throw new Error(`No agent-manifest.json found in ${source}`); - } - - const manifest = JSON.parse( - fs.readFileSync(manifestPath, 'utf-8') - ) as Manifest; - cached = manifest; - cachedSourceRoot = tmpDir; - return { manifest, sourceRoot: tmpDir }; -} - -export function getSourceRoot(): string { - if (cachedSourceRoot) return cachedSourceRoot; - loadManifest(); - return cachedSourceRoot!; -} - -export function getTemplates(): TemplateEntry[] { - return (cached ?? loadManifest()).templates; -} - -export function getTemplate(id: string): TemplateEntry | undefined { - return getTemplates().find((t) => t.id === id); -} - -export function getProviders(): ProviderEntry[] { - return (cached ?? loadManifest()).providers; -} diff --git a/src/utils/agent-scaffold/scaffold.ts b/src/utils/agent-scaffold/scaffold.ts deleted file mode 100644 index 6e0667a8e2..0000000000 --- a/src/utils/agent-scaffold/scaffold.ts +++ /dev/null @@ -1,246 +0,0 @@ -/** - * Vendored from firecrawl/web-agent:.internal/cli/src/utils/scaffold.ts - * Copies templates from the cloned agent repo into the user's project dir, - * then merges agent-core deps and writes .env. No divergence from upstream - * — keep in sync if upstream changes scaffolding behavior. - */ - -import * as fs from 'fs'; -import * as path from 'path'; -import { execSync } from 'child_process'; -import type { TemplateEntry } from './manifest'; -import { getSourceRoot } from './manifest'; -import { success, info, warn } from './ui'; - -const SKIP = new Set([ - 'node_modules', - '.next', - '.git', - '.DS_Store', - 'README.md', -]); - -/** Files to skip when copying agent-core (dev artifacts, build config) */ -const AGENT_CORE_SKIP = new Set([ - 'dist', - 'tsup.config.ts', - 'tsconfig.json', - 'package.json', -]); - -function copyDirRecursive( - src: string, - dest: string, - extraSkip?: (name: string) => boolean -): void { - if (!fs.existsSync(src)) return; - fs.mkdirSync(dest, { recursive: true }); - - for (const entry of fs.readdirSync(src, { withFileTypes: true })) { - if (SKIP.has(entry.name)) continue; - if (extraSkip?.(entry.name)) continue; - const srcPath = path.join(src, entry.name); - // Skip symlinks — agent-core is copied separately - try { - if (fs.lstatSync(srcPath).isSymbolicLink()) continue; - } catch { - /* proceed */ - } - const destPath = path.join(dest, entry.name); - - if (entry.isDirectory()) { - copyDirRecursive(srcPath, destPath, extraSkip); - } else { - fs.copyFileSync(srcPath, destPath); - } - } -} - -/** Clean up any leftover workspace references in scaffolded files. */ -function cleanupScaffold(dir: string): void { - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const fullPath = path.join(dir, entry.name); - if (entry.isDirectory() && !SKIP.has(entry.name)) { - cleanupScaffold(fullPath); - } else if (entry.name === 'package.json') { - const pkg = JSON.parse(fs.readFileSync(fullPath, 'utf-8')); - if (pkg.dependencies?.['@firecrawl/agent-core']) { - delete pkg.dependencies['@firecrawl/agent-core']; - fs.writeFileSync( - fullPath, - JSON.stringify(pkg, null, 2) + '\n', - 'utf-8' - ); - } - } - } -} - -function generateEnvFile(keys: Record): string { - const lines: string[] = []; - for (const [envVar, value] of Object.entries(keys)) { - if (value) lines.push(`${envVar}=${value}`); - } - return lines.join('\n') + '\n'; -} - -export interface ScaffoldOptions { - projectDir: string; - template: TemplateEntry; - envVars: Record; - selectedProvider?: string; - defaultModelId?: string; - /** Sub-agent provider. Defaults to selectedProvider when omitted. */ - subAgentProvider?: string; - /** Sub-agent model id. Defaults to defaultModelId when omitted. */ - subAgentModelId?: string; - skipInstall?: boolean; -} - -function applyTemplateProviderDefaults( - projectDir: string, - templateId: string, - orchProvider: string | undefined, - orchModel: string | undefined, - subProvider: string | undefined, - subModel: string | undefined -): void { - if (!orchProvider || !orchModel || templateId !== 'next') return; - - const configPath = path.join(projectDir, 'app', '(agent)', '_config.ts'); - if (!fs.existsSync(configPath)) return; - - const effectiveSubProvider = subProvider ?? orchProvider; - const effectiveSubModel = subModel ?? orchModel; - - let content = fs.readFileSync(configPath, 'utf-8'); - content = content - .replace( - /^(\s*)orchestrator:\s*\{ provider: "[^"]+", model: "[^"]+" \} satisfies ModelRef,$/m, - `$1orchestrator: { provider: "${orchProvider}", model: "${orchModel}" } satisfies ModelRef,` - ) - .replace( - /^(\s*)subAgent:\s*\{ provider: "[^"]+", model: "[^"]+" \} satisfies ModelRef,$/m, - `$1subAgent: { provider: "${effectiveSubProvider}", model: "${effectiveSubModel}" } satisfies ModelRef,` - ) - .replace( - /^(\s*)background:\s*\{ provider: "[^"]+", model: "[^"]+" \} satisfies ModelRef,$/m, - `$1background: { provider: "${orchProvider}", model: "${orchModel}" } satisfies ModelRef,` - ); - - fs.writeFileSync(configPath, content, 'utf-8'); -} - -export async function scaffoldProject(opts: ScaffoldOptions): Promise { - const { - projectDir, - template, - envVars, - selectedProvider, - defaultModelId, - subAgentProvider, - subAgentModelId, - skipInstall, - } = opts; - const sourceRoot = getSourceRoot(); - - fs.mkdirSync(projectDir, { recursive: true }); - - // Copy agent-core (skip dev artifacts, test files, build output) - const agentCoreSrc = path.join(sourceRoot, 'agent-core'); - if (fs.existsSync(agentCoreSrc)) { - copyDirRecursive( - agentCoreSrc, - path.join(projectDir, 'agent-core'), - (name) => { - if (AGENT_CORE_SKIP.has(name)) return true; - if (name.endsWith('.test.ts')) return true; - return false; - } - ); - } - - // Copy template files - const templateSrc = path.join(sourceRoot, template.path); - for (const entry of fs.readdirSync(templateSrc, { withFileTypes: true })) { - if (SKIP.has(entry.name)) continue; - const srcPath = path.join(templateSrc, entry.name); - // Skip symlinks (agent-core is already copied above) - try { - if (fs.lstatSync(srcPath).isSymbolicLink()) continue; - } catch { - /* proceed */ - } - const destPath = path.join(projectDir, entry.name); - - if (entry.isDirectory()) { - copyDirRecursive(srcPath, destPath); - } else { - fs.copyFileSync(srcPath, destPath); - } - } - - // Clean up any leftover workspace references - cleanupScaffold(projectDir); - - // Merge agent-core deps into the project package.json - const agentCorePkgPath = path.join(sourceRoot, 'agent-core', 'package.json'); - const projectPkgPath = path.join(projectDir, 'package.json'); - if (fs.existsSync(agentCorePkgPath) && fs.existsSync(projectPkgPath)) { - const corePkg = JSON.parse(fs.readFileSync(agentCorePkgPath, 'utf-8')); - const projectPkg = JSON.parse(fs.readFileSync(projectPkgPath, 'utf-8')); - // Add agent-core runtime deps to the project - for (const [dep, version] of Object.entries(corePkg.dependencies ?? {})) { - if (!projectPkg.dependencies?.[dep]) { - projectPkg.dependencies = projectPkg.dependencies ?? {}; - projectPkg.dependencies[dep] = version; - } - } - // Add agent-core peer deps (optional providers) - for (const [dep, version] of Object.entries( - corePkg.peerDependencies ?? {} - )) { - if (!projectPkg.dependencies?.[dep]) { - projectPkg.dependencies = projectPkg.dependencies ?? {}; - projectPkg.dependencies[dep] = version; - } - } - fs.writeFileSync( - projectPkgPath, - JSON.stringify(projectPkg, null, 2) + '\n', - 'utf-8' - ); - } - - applyTemplateProviderDefaults( - projectDir, - template.id, - selectedProvider, - defaultModelId, - subAgentProvider, - subAgentModelId - ); - success(`${template.name} template scaffolded`); - - // Write .env file - const envFileName = template.envFile ?? '.env'; - const envPath = path.join(projectDir, envFileName); - fs.writeFileSync(envPath, generateEnvFile(envVars), 'utf-8'); - success(`Created ${envFileName}`); - - // Install dependencies - if (!skipInstall) { - info('Installing dependencies...'); - try { - execSync('npm install', { cwd: projectDir, stdio: 'pipe' }); - success('Dependencies installed'); - } catch (err) { - const stderr = - err instanceof Error && 'stderr' in err - ? (err as { stderr: Buffer }).stderr?.toString().trim() - : ''; - warn('npm install failed — run it manually in the project directory'); - if (stderr) console.error(`\n${stderr}\n`); - } - } -} diff --git a/src/utils/agent-scaffold/ui.ts b/src/utils/agent-scaffold/ui.ts deleted file mode 100644 index f4fb39c138..0000000000 --- a/src/utils/agent-scaffold/ui.ts +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Vendored from firecrawl/web-agent:.internal/cli/src/utils/ui.ts - * Keep in sync if upstream changes — this file intentionally mirrors the - * agent repo's CLI output style so scaffold messages match docs verbatim. - */ - -import { - bold, - cyan, - dim, - green, - orange, - red, - reset, - yellow, -} from '../terminal'; - -export { bold, cyan, dim, green, orange, red, reset, yellow }; - -export function printBanner(): void { - console.log(''); - console.log(` ${orange}${bold}firecrawl-agent${reset}`); - console.log(` ${dim}AI-powered web research agent${reset}`); - console.log(''); -} - -export function step(n: number, label: string): void { - console.log(` ${dim}${n}.${reset} ${label}`); -} - -export function success(msg: string): void { - console.log(` ${green}✓${reset} ${msg}`); -} - -export function warn(msg: string): void { - console.log(` ${yellow}!${reset} ${msg}`); -} - -export function error(msg: string): void { - console.log(` ${red}✗${reset} ${msg}`); -} - -export function info(msg: string): void { - console.log(` ${dim}${msg}${reset}`); -}