diff --git a/README.md b/README.md index 9cf107826a..e0c25f714c 100644 --- a/README.md +++ b/README.md @@ -1325,4 +1325,4 @@ firecrawl alexandria feedback --rating partial \ --rationale "Found summaries but could not retrieve attachments" --json ``` -The optional `--objective` is the underlying goal of the session: what you or your user were ultimately trying to accomplish, beyond the single website. No job ID is required. It must be sent within 20 minutes of the team's most recent Alexandria search, discovery, or execution; later submissions are rejected with `FEEDBACK_WINDOW_EXPIRED`. Each submission refunds 1 credit, up to 10 credits per website and 100 credits per team each UTC day; past either cap, submissions are still recorded and return `websiteCapReached` or `dailyCapReached`. Optional `--provider-feedback` and `--capability-feedback` accept JSON arrays; see `firecrawl alexandria feedback --help` for their fields and issue codes. Capability issue codes are `new_capability_request` (requires `requestedFunctionality`), `missing_capability` (the provider exists but lacks this capability), `insufficient_functionality`, `incorrect_result`, `execution_error`, and `other`. Existing `feedback` and `search-feedback` commands retain their job-specific behavior. Endpoint feedback opt-out environment variables also apply to this command. +The optional `--objective` is the underlying goal of the session: what you or your user were ultimately trying to accomplish, beyond the single website. No job ID is required. It must be sent within 20 minutes of the team's most recent Alexandria search, discovery, or execution; later submissions are rejected with `FEEDBACK_WINDOW_EXPIRED`. Each submission refunds 1 credit, up to 10 credits per website and 100 credits per team each UTC day; past either cap, submissions are still recorded and return `websiteCapReached` or `dailyCapReached`. Optional `--provider-feedback` and `--capability-feedback` accept JSON arrays; see `firecrawl alexandria feedback --help` for their fields and issue codes. Capability issue codes are `new_capability_request` (requires `requestedFunctionality`), `missing_capability` (the provider exists but lacks this capability), `insufficient_functionality`, `incorrect_result`, `execution_error`, and `other`. Each eligible provider call prints a `Feedback ref` (for example `c3`) so ratings are attributed to the calls they are about. The CLI keeps up to 20 call tokens for 24 hours in its config directory and sends them with the session rating, then clears them after a successful submission. Rate individual calls with a repeatable `--call-rating c3=good`, or pass tokens yourself with a repeatable `--token ` (these replace the stored ones). Existing `feedback` and `search-feedback` commands retain their job-specific behavior. Endpoint feedback opt-out environment variables also apply to this command. diff --git a/package.json b/package.json index 3b0f1a8c9f..0ba2b8b4c9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "firecrawl-cli", - "version": "1.28.0", + "version": "1.29.0", "publishConfig": { "tag": "latest" }, diff --git a/src/__tests__/commands/alexandria-feedback-tokens.test.ts b/src/__tests__/commands/alexandria-feedback-tokens.test.ts new file mode 100644 index 0000000000..c1bd675913 --- /dev/null +++ b/src/__tests__/commands/alexandria-feedback-tokens.test.ts @@ -0,0 +1,261 @@ +import { mkdtempSync, rmSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const state = vi.hoisted(() => ({ dir: '' })); + +vi.mock('../../utils/credentials', async () => ({ + ...(await vi.importActual('../../utils/credentials')), + getConfigDirectoryPath: () => state.dir, +})); + +vi.mock('../../utils/agent-origin', () => ({ + getCliOrigin: () => 'cli.test@1.0.0', +})); + +vi.mock('../../utils/client', async () => ({ + ...(await vi.importActual('../../utils/client')), + getClient: vi.fn(), +})); + +import { requestAlexandria } from '../../commands/alexandria'; +import { + executeEndpointFeedback, + handleEndpointFeedbackCommand, + parseCallRatingArg, + parseFeedbackTokenArg, +} from '../../commands/feedback'; +import { getClient } from '../../utils/client'; +import { initializeConfig } from '../../utils/config'; +import { + feedbackTokenScope, + readFeedbackTokens, + recordFeedbackTokens, +} from '../../utils/alexandria-feedback-tokens'; +import { setupTest, teardownTest } from '../utils/mock-client'; + +const scope = feedbackTokenScope('test-api-key'); +const base = { + endpoint: 'alexandria' as const, + rating: 'good' as const, + requestedWebsite: { + url: 'https://example.com', + requestedFunctionality: 'Read records', + }, + rationale: 'Worked', +}; + +describe('alexandria feedback tokens in the CLI', () => { + let mockFetch: ReturnType; + + beforeEach(() => { + state.dir = mkdtempSync(join(tmpdir(), 'fc-cli-tokens-')); + setupTest(); + initializeConfig({ + apiKey: 'test-api-key', + apiUrl: 'https://api.firecrawl.dev', + }); + mockFetch = vi.fn().mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ + success: true, + feedbackId: 'fb', + creditsRefunded: 1, + }), + }); + global.fetch = mockFetch as unknown as typeof fetch; + }); + + afterEach(() => { + teardownTest(); + vi.clearAllMocks(); + rmSync(state.dir, { recursive: true, force: true }); + }); + + const sentBody = () => JSON.parse(mockFetch.mock.calls[0][1].body); + + it('records tokens from Alexandria results and prints a ref per call', async () => { + const errors = vi.spyOn(console, 'error').mockImplementation(() => {}); + vi.mocked(getClient).mockReturnValue({ + scrape: vi.fn().mockResolvedValue({ + creditsCost: 1, + alexandria: [ + { + provider: 'fred', + capability: 'series', + feedbackToken: 'tok-1', + data: {}, + }, + { + provider: 'fred', + capability: 'bad', + feedbackToken: 'tok-2', + error: { code: 'x' }, + }, + { + provider: 'firecrawl', + capability: 'find-tools', + feedbackToken: 'tok-3', + data: {}, + }, + { provider: 'fred', capability: 'plain', data: {} }, + ], + }), + } as any); + const envelope = await requestAlexandria( + [ + { provider: 'fred', capability: 'series', options: {} }, + { provider: 'fred', capability: 'bad', options: {} }, + { provider: 'firecrawl', capability: 'find-tools', options: {} }, + { provider: 'fred', capability: 'plain', options: {} }, + ], + {} + ); + expect(readFeedbackTokens(scope).map((entry) => entry.token)).toEqual([ + 'tok-1', + 'tok-2', + ]); + const lines = errors.mock.calls.map((call) => String(call[0])); + expect(lines).toContain('Feedback ref c1: fred/series'); + expect(lines).toContain('Feedback ref c2: fred/bad'); + expect(JSON.stringify(lines)).not.toContain('tok-'); + expect(envelope.data.alexandria[0].feedbackToken).toBe('tok-1'); + errors.mockRestore(); + }); + + it('sends the stored tokens and clears them on success', async () => { + recordFeedbackTokens(scope, ['tok-1', 'tok-2']); + const result = await executeEndpointFeedback(base); + expect(result.success).toBe(true); + expect(sentBody().tokens).toEqual(['tok-1', 'tok-2']); + expect(sentBody().callRatings).toBeUndefined(); + expect(readFeedbackTokens(scope)).toEqual([]); + }); + + it('keeps the tokens when the submission fails', async () => { + recordFeedbackTokens(scope, ['tok-1']); + mockFetch.mockResolvedValue({ + ok: false, + status: 400, + statusText: 'Bad Request', + json: async () => ({ success: false, error: 'nope' }), + }); + expect((await executeEndpointFeedback(base)).success).toBe(false); + expect(readFeedbackTokens(scope)).toHaveLength(1); + }); + + it('maps call ratings from refs to tokens', async () => { + recordFeedbackTokens(scope, ['tok-1', 'tok-2']); + await executeEndpointFeedback({ + ...base, + callRatings: [ + { ref: 'c1', rating: 'good' }, + { ref: 'c2', rating: 'bad' }, + ], + }); + expect(sentBody().callRatings).toEqual([ + { token: 'tok-1', rating: 'good' }, + { token: 'tok-2', rating: 'bad' }, + ]); + expect(sentBody().tokens).toEqual(['tok-1', 'tok-2']); + }); + + it('refuses an unknown call ref without sending anything', async () => { + const result = await executeEndpointFeedback({ + ...base, + callRatings: [{ ref: 'c9', rating: 'bad' }], + }); + expect(result.success).toBe(false); + expect(result.error).toContain('Unknown call ref "c9"'); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('sends explicit tokens in place of the stored ones', async () => { + recordFeedbackTokens(scope, ['stored']); + await executeEndpointFeedback({ + ...base, + tokens: ['explicit-1', 'explicit-2'], + callRatings: [{ ref: 'explicit-2', rating: 'partial' }], + }); + expect(sentBody().tokens).toEqual(['explicit-1', 'explicit-2']); + expect(sentBody().callRatings).toEqual([ + { token: 'explicit-2', rating: 'partial' }, + ]); + expect(readFeedbackTokens(scope).map((entry) => entry.token)).toEqual([ + 'stored', + ]); + }); + + it('resolves call-rating refs only against explicit tokens when given', async () => { + recordFeedbackTokens(scope, ['stored']); + const result = await executeEndpointFeedback({ + ...base, + tokens: ['explicit'], + callRatings: [{ ref: 'c1', rating: 'bad' }], + }); + expect(result.success).toBe(false); + expect(result.error).toContain('Unknown call ref "c1"'); + expect(mockFetch).not.toHaveBeenCalled(); + expect(readFeedbackTokens(scope)).toHaveLength(1); + }); + + it('sends no token fields when there are none', async () => { + await executeEndpointFeedback(base); + expect(sentBody().tokens).toBeUndefined(); + expect(sentBody().callRatings).toBeUndefined(); + }); + + it('does not send tokens for job feedback', async () => { + recordFeedbackTokens(scope, ['tok-1']); + await executeEndpointFeedback({ + endpoint: 'scrape', + jobId: '0193f6c5-1234-7890-abcd-1234567890ab', + rating: 'good', + }); + expect(sentBody().tokens).toBeUndefined(); + expect(readFeedbackTokens(scope)).toHaveLength(1); + }); + + it('reports rejected tokens from the API', async () => { + recordFeedbackTokens(scope, ['tok-1']); + mockFetch.mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ + success: true, + feedbackId: 'fb', + rejectedTokens: 1, + }), + }); + expect((await executeEndpointFeedback(base)).rejectedTokens).toBe(1); + + const stdout = vi + .spyOn(process.stdout, 'write') + .mockImplementation(() => true); + try { + await handleEndpointFeedbackCommand(base); + const output = stdout.mock.calls.map(([chunk]) => chunk).join(''); + expect(output).toContain( + '1 call token(s) could not be verified and were ignored.' + ); + } finally { + stdout.mockRestore(); + } + }); + + it('parses call ratings and tokens', () => { + expect(parseCallRatingArg('c3=good')).toEqual([ + { ref: 'c3', rating: 'good' }, + ]); + expect( + parseCallRatingArg('c4=BAD', parseCallRatingArg('c3=good')) + ).toHaveLength(2); + expect(() => parseCallRatingArg('c3')).toThrow('--call-rating'); + expect(() => parseCallRatingArg('=good')).toThrow('--call-rating'); + expect(() => parseCallRatingArg('c3=great')).toThrow('--rating'); + expect(parseFeedbackTokenArg('abc==')).toEqual(['abc==']); + expect(() => parseFeedbackTokenArg('x'.repeat(1025))).toThrow('1-1024'); + }); +}); diff --git a/src/__tests__/commands/feedback.test.ts b/src/__tests__/commands/feedback.test.ts index 106ad01731..5151018e0f 100644 --- a/src/__tests__/commands/feedback.test.ts +++ b/src/__tests__/commands/feedback.test.ts @@ -1,4 +1,13 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { rmSync } from 'fs'; +import { + describe, + it, + expect, + vi, + beforeEach, + afterEach, + afterAll, +} from 'vitest'; import { executeEndpointFeedback, handleEndpointFeedbackCommand, @@ -16,6 +25,19 @@ import { getClient } from '../../utils/client'; import { initializeConfig } from '../../utils/config'; import { setupTest, teardownTest } from '../utils/mock-client'; +afterAll(() => rmSync(configDir, { recursive: true, force: true })); + +const configDir = vi.hoisted(() => + require('fs').mkdtempSync( + require('path').join(require('os').tmpdir(), 'fc-feedback-') + ) +); + +vi.mock('../../utils/credentials', async () => ({ + ...(await vi.importActual('../../utils/credentials')), + getConfigDirectoryPath: () => configDir, +})); + vi.mock('../../utils/agent-origin', () => ({ getCliOrigin: () => 'cli.test@1.0.0', })); diff --git a/src/__tests__/utils/alexandria-feedback-tokens.test.ts b/src/__tests__/utils/alexandria-feedback-tokens.test.ts new file mode 100644 index 0000000000..f2531542e0 --- /dev/null +++ b/src/__tests__/utils/alexandria-feedback-tokens.test.ts @@ -0,0 +1,133 @@ +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + utimesSync, + writeFileSync, +} from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const state = vi.hoisted(() => ({ dir: '', root: '' })); + +vi.mock('../../utils/credentials', async () => ({ + ...(await vi.importActual('../../utils/credentials')), + getConfigDirectoryPath: () => state.dir, +})); + +import { + FEEDBACK_TOKEN_LIMIT, + FEEDBACK_TOKEN_TTL_MS, + clearFeedbackTokens, + feedbackTokenScope, + readFeedbackTokens, + recordFeedbackTokens, +} from '../../utils/alexandria-feedback-tokens'; + +const scope = feedbackTokenScope('fc-key'); + +describe('alexandria feedback token store', () => { + beforeEach(() => { + state.root = mkdtempSync(join(tmpdir(), 'fc-tokens-')); + state.dir = state.root; + }); + + afterEach(() => { + rmSync(state.root, { recursive: true, force: true }); + }); + + it('keeps a token on one ref across calls', () => { + expect(recordFeedbackTokens(scope, ['a', 'b'])).toEqual(['c1', 'c2']); + expect(recordFeedbackTokens(scope, ['b', 'c'])).toEqual(['c2', 'c3']); + expect(readFeedbackTokens(scope).map((entry) => entry.token)).toEqual([ + 'a', + 'b', + 'c', + ]); + }); + + it('keeps the newest tokens up to the limit', () => { + recordFeedbackTokens( + scope, + Array.from({ length: FEEDBACK_TOKEN_LIMIT + 3 }, (_, i) => `t${i}`) + ); + const tokens = readFeedbackTokens(scope).map((entry) => entry.token); + expect(tokens).toHaveLength(FEEDBACK_TOKEN_LIMIT); + expect(tokens[0]).toBe('t3'); + }); + + it('drops tokens after 24 hours', () => { + const start = 1_000_000; + recordFeedbackTokens(scope, ['old'], start); + recordFeedbackTokens(scope, ['new'], start + FEEDBACK_TOKEN_TTL_MS - 1); + expect( + readFeedbackTokens(scope, start + FEEDBACK_TOKEN_TTL_MS + 1).map( + (entry) => entry.token + ) + ).toEqual(['new']); + }); + + it('keeps each credential tokens apart', () => { + recordFeedbackTokens(scope, ['mine']); + recordFeedbackTokens(feedbackTokenScope('fc-other'), ['theirs']); + expect(readFeedbackTokens(scope).map((entry) => entry.token)).toEqual([ + 'mine', + ]); + }); + + it('returns no ref for tokens the cap dropped', () => { + const tokens = Array.from( + { length: FEEDBACK_TOKEN_LIMIT + 2 }, + (_, i) => `t${i}` + ); + const refs = recordFeedbackTokens(scope, tokens); + expect(refs.slice(0, 2)).toEqual(['', '']); + expect(refs.slice(2).every(Boolean)).toBe(true); + }); + + it('removes expired tokens from disk on read', () => { + const start = 1_000_000; + recordFeedbackTokens(scope, ['old'], start); + expect( + readFeedbackTokens(scope, start + FEEDBACK_TOKEN_TTL_MS + 1) + ).toEqual([]); + const file = JSON.parse( + readFileSync(join(state.dir, 'alexandria-feedback-tokens.json'), 'utf-8') + ); + expect(file.entries).toEqual([]); + }); + + it('breaks a stale lock left by a crashed process', () => { + const lock = join(state.dir, 'alexandria-feedback-tokens.lock'); + mkdirSync(lock); + const old = new Date(Date.now() - 60_000); + utimesSync(lock, old, old); + expect(recordFeedbackTokens(scope, ['a'])).toEqual(['c1']); + }); + + it('clears only the sent tokens', () => { + recordFeedbackTokens(scope, ['a', 'b', 'c']); + clearFeedbackTokens(scope, ['a', 'c']); + expect(readFeedbackTokens(scope).map((entry) => entry.token)).toEqual([ + 'b', + ]); + }); + + it('writes the file with mode 0600', () => { + recordFeedbackTokens(scope, ['a']); + const mode = + statSync(join(state.dir, 'alexandria-feedback-tokens.json')).mode & 0o777; + if (process.platform !== 'win32') expect(mode).toBe(0o600); + }); + + it('never throws when the store cannot be written', () => { + const blocker = join(state.root, 'blocker'); + writeFileSync(blocker, ''); + state.dir = join(blocker, 'config'); + expect(recordFeedbackTokens(scope, ['a'])).toEqual([]); + expect(() => clearFeedbackTokens(scope, ['a'])).not.toThrow(); + }); +}); diff --git a/src/commands/alexandria-feedback.ts b/src/commands/alexandria-feedback.ts index ab67defc3c..f7a7821b8a 100644 --- a/src/commands/alexandria-feedback.ts +++ b/src/commands/alexandria-feedback.ts @@ -1,7 +1,9 @@ import { Command, InvalidArgumentError } from 'commander'; import { handleEndpointFeedbackCommand, + parseCallRatingArg, parseEndpointFeedbackRating, + parseFeedbackTokenArg, } from './feedback'; function detail(value: string): string { @@ -124,6 +126,16 @@ export function createAlexandriaFeedbackCommand(): Command { 'Array of {name, provider, issue, why, requestedFunctionality?}; issues: new_capability_request (requires requestedFunctionality), missing_capability (provider exists but lacks this capability), insufficient_functionality, incorrect_result, execution_error, other', (value) => parseAlexandriaFeedbackArray(value, true) ) + .option( + '--call-rating ', + 'Rate one call by the ref printed after it, for example c3=good (repeatable). The overall --rating covers every call in the session', + parseCallRatingArg + ) + .option( + '--token ', + 'Send this call feedback token instead of the stored ones (repeatable, at most 20), for scripted use', + parseFeedbackTokenArg + ) .option('-k, --api-key ', 'Firecrawl API key') .option('--api-url ', 'API base URL') .option('-o, --output ', 'Save the response to a file') @@ -133,6 +145,8 @@ export function createAlexandriaFeedbackCommand(): Command { .action(async (options) => { await handleEndpointFeedbackCommand({ ...options, + tokens: options.token, + callRatings: options.callRating, endpoint: 'alexandria', requestedWebsite: { url: options.url, diff --git a/src/commands/alexandria.ts b/src/commands/alexandria.ts index c246e5cd46..55f8ff74a1 100644 --- a/src/commands/alexandria.ts +++ b/src/commands/alexandria.ts @@ -7,6 +7,10 @@ import { getClient } from '../utils/client'; import { getApiKey } from '../utils/config'; import { writeOutput } from '../utils/output'; import { withoutResultPrices } from '../utils/alexandria-prices'; +import { + feedbackTokenScope, + recordFeedbackTokens, +} from '../utils/alexandria-feedback-tokens'; import { TERMS_DASHBOARD_URL } from './terms'; type Call = AlexandriaCall & { options: Record }; @@ -226,6 +230,23 @@ export async function requestAlexandria( creditsCost: result.creditsCost, }, }; + const tokened = ( + Array.isArray(result.alexandria) ? result.alexandria : [] + ).filter( + (item: any) => + typeof item?.feedbackToken === 'string' && item.provider !== 'firecrawl' + ); + const refs = recordFeedbackTokens( + feedbackTokenScope(getApiKey(options.apiKey)), + tokened.map((item: any) => item.feedbackToken as string) + ); + if (options.showReceipt !== false) + tokened.forEach((item, i) => { + if (refs[i]) + console.error( + `Feedback ref ${refs[i]}: ${item.provider}/${item.capability}` + ); + }); } catch (error) { envelope = apiFailure(error); } diff --git a/src/commands/feedback.ts b/src/commands/feedback.ts index dd7eb37fbd..f9e2b606c9 100644 --- a/src/commands/feedback.ts +++ b/src/commands/feedback.ts @@ -13,6 +13,12 @@ import { type ValuableSourceInput, } from './search-feedback'; import { integrationHeaders } from '../utils/integration'; +import { + FEEDBACK_TOKEN_LIMIT, + clearFeedbackTokens, + feedbackTokenScope, + readFeedbackTokens, +} from '../utils/alexandria-feedback-tokens'; export type EndpointFeedbackEndpoint = 'search' | 'scrape' | 'parse' | 'map'; @@ -24,6 +30,8 @@ export interface EndpointFeedbackOptions { objective?: string; providerFeedback?: Record[]; capabilityFeedback?: Record[]; + tokens?: string[]; + callRatings?: { ref: string; rating: SearchFeedbackRating }[]; rating: SearchFeedbackRating; issues?: string[]; tags?: string[]; @@ -63,6 +71,7 @@ export interface EndpointFeedbackResult { dailyCapReached?: boolean; websiteCapReached?: boolean; alreadySubmitted?: boolean; + rejectedTokens?: number; warning?: string; error?: string; errorCode?: EndpointFeedbackErrorCode; @@ -199,6 +208,75 @@ export function parseEndpointFeedbackRating( return rating as SearchFeedbackRating; } +/** + * The call tokens an Alexandria rating covers: the explicit ones when given, else those + * stored by earlier calls. Call-rating refs resolve against the store (or an explicit token). + */ +function resolveAlexandriaTokens( + options: EndpointFeedbackOptions, + scope: string +) { + const stored = readFeedbackTokens(scope); + const explicit = options.tokens ?? []; + const byRef = new Map( + explicit.length + ? explicit.map((token): [string, string] => [token, token]) + : stored.map((entry): [string, string] => [entry.ref, entry.token]) + ); + const rated = new Map(); + for (const { ref, rating } of options.callRatings ?? []) { + const token = byRef.get(ref); + if (!token) { + throw new Error( + `Unknown call ref "${ref}". Refs are printed after each Alexandria call and expire after 24 hours.` + ); + } + rated.set(token, rating); + } + const tokens = [ + ...new Set(explicit.length ? explicit : stored.map((entry) => entry.token)), + ].slice(-FEEDBACK_TOKEN_LIMIT); + return { + tokens, + callRatings: [...rated].map(([token, rating]) => ({ token, rating })), + sent: [...new Set([...tokens, ...rated.keys()])], + }; +} + +export function parseCallRatingArg( + value: string, + previous: { ref: string; rating: SearchFeedbackRating }[] = [] +): { ref: string; rating: SearchFeedbackRating }[] { + const separator = value.lastIndexOf('='); + const ref = value.slice(0, separator).trim(); + if (separator < 1 || !ref) { + throw new Error('--call-rating must look like ='); + } + if (previous.length >= FEEDBACK_TOKEN_LIMIT) { + throw new Error(`Rate at most ${FEEDBACK_TOKEN_LIMIT} calls.`); + } + return [ + ...previous, + { + ref, + rating: parseEndpointFeedbackRating(value.slice(separator + 1).trim()), + }, + ]; +} + +export function parseFeedbackTokenArg( + value: string, + previous: string[] = [] +): string[] { + if (!value || value.length > 1024) { + throw new Error('--token must contain 1-1024 characters.'); + } + if (previous.length >= FEEDBACK_TOKEN_LIMIT) { + throw new Error(`Pass at most ${FEEDBACK_TOKEN_LIMIT} tokens.`); + } + return [...previous, value]; +} + export function isEndpointFeedbackDisabledLocally( env: NodeJS.ProcessEnv = process.env ): boolean { @@ -281,6 +359,11 @@ export async function executeEndpointFeedback( '--valuable-results is only supported for search feedback.' ); } + const scope = feedbackTokenScope(apiKey); + const attribution = + options.endpoint === 'alexandria' + ? resolveAlexandriaTokens(options, scope) + : undefined; const entries: Array<[string, unknown]> = options.endpoint === 'alexandria' ? [ @@ -289,6 +372,8 @@ export async function executeEndpointFeedback( ['objective', options.objective], ['providerFeedback', options.providerFeedback], ['capabilityFeedback', options.capabilityFeedback], + ['tokens', attribution?.tokens], + ['callRatings', attribution?.callRatings], ] : [ ['issues', normalizeList(options.issues)], @@ -352,6 +437,8 @@ export async function executeEndpointFeedback( }; } + if (attribution) clearFeedbackTokens(scope, attribution.sent); + return { success: true, feedbackId: data.feedbackId, @@ -367,6 +454,9 @@ export async function executeEndpointFeedback( dailyCapReached: data.dailyCapReached === true, ...(data.websiteCapReached === true ? { websiteCapReached: true } : {}), alreadySubmitted: data.alreadySubmitted, + ...(typeof data.rejectedTokens === 'number' + ? { rejectedTokens: data.rejectedTokens } + : {}), warning: data.warning, }; } catch (error: any) { @@ -405,6 +495,11 @@ function formatReadable(result: EndpointFeedbackResult): string { 'Daily refund cap reached for this website; feedback about other websites can still refund credits.' ); } + if (result.rejectedTokens) { + lines.push( + `${result.rejectedTokens} call token(s) could not be verified and were ignored.` + ); + } if (result.warning) { lines.push(`Warning: ${result.warning}`); } @@ -457,6 +552,9 @@ export async function handleEndpointFeedbackCommand( ...(result.dailyCapReached ? { dailyCapReached: true } : {}), ...(result.websiteCapReached ? { websiteCapReached: true } : {}), ...(result.alreadySubmitted ? { alreadySubmitted: true } : {}), + ...(result.rejectedTokens + ? { rejectedTokens: result.rejectedTokens } + : {}), ...(result.warning ? { warning: result.warning } : {}), }; outputContent = options.pretty diff --git a/src/utils/alexandria-feedback-tokens.ts b/src/utils/alexandria-feedback-tokens.ts new file mode 100644 index 0000000000..637eff7fdb --- /dev/null +++ b/src/utils/alexandria-feedback-tokens.ts @@ -0,0 +1,189 @@ +/** + * Local store of Alexandria call feedback tokens, so a later `firecrawl alexandria feedback` + * process can rate the calls an earlier process made. + */ + +import { createHash } from 'crypto'; +import { + chmodSync, + existsSync, + mkdirSync, + readFileSync, + renameSync, + rmdirSync, + statSync, + writeFileSync, +} from 'fs'; +import { join } from 'path'; +import { getConfigDirectoryPath } from './credentials'; + +export const FEEDBACK_TOKEN_LIMIT = 20; +export const FEEDBACK_TOKEN_TTL_MS = 24 * 60 * 60 * 1000; +const LOCK_WAIT_MS = 2000; +const LOCK_STALE_MS = 10_000; + +export interface StoredFeedbackToken { + ref: string; + token: string; + scope: string; + at: number; +} + +interface StoreFile { + counter: number; + entries: StoredFeedbackToken[]; +} + +function storePath(): string { + return join(getConfigDirectoryPath(), 'alexandria-feedback-tokens.json'); +} + +/** Tokens belong to one team, so they are only offered back to the credential that received them. */ +export function feedbackTokenScope(apiKey?: string): string { + return createHash('sha256') + .update(apiKey ?? '') + .digest('hex') + .slice(0, 16); +} + +function load(now: number): { store: StoreFile; pruned: boolean } { + try { + const parsed = JSON.parse(readFileSync(storePath(), 'utf-8')); + const all: StoredFeedbackToken[] = Array.isArray(parsed?.entries) + ? parsed.entries + : []; + const entries = all.filter( + (entry) => + typeof entry?.ref === 'string' && + typeof entry.token === 'string' && + typeof entry.scope === 'string' && + typeof entry.at === 'number' && + entry.at > now - FEEDBACK_TOKEN_TTL_MS + ); + return { + store: { + counter: Number.isInteger(parsed?.counter) ? parsed.counter : 0, + entries, + }, + pruned: entries.length !== all.length, + }; + } catch { + return { store: { counter: 0, entries: [] }, pruned: false }; + } +} + +function save(store: StoreFile): void { + const path = storePath(); + const dir = getConfigDirectoryPath(); + if (!existsSync(dir)) mkdirSync(dir, { recursive: true, mode: 0o700 }); + const temporary = `${path}.${process.pid}.tmp`; + writeFileSync(temporary, JSON.stringify(store), { mode: 0o600 }); + chmodSync(temporary, 0o600); + renameSync(temporary, path); +} + +/** Serializes read-modify-write across processes with an exclusive lock directory. */ +function withLock(run: () => T): T { + const dir = getConfigDirectoryPath(); + if (!existsSync(dir)) mkdirSync(dir, { recursive: true, mode: 0o700 }); + const lock = join(dir, 'alexandria-feedback-tokens.lock'); + const deadline = Date.now() + LOCK_WAIT_MS; + for (;;) { + try { + mkdirSync(lock); + break; + } catch (error: any) { + if (error?.code !== 'EEXIST') throw error; + try { + if (Date.now() - statSync(lock).mtimeMs > LOCK_STALE_MS) + rmdirSync(lock); + } catch { + // Released by its owner in the meantime. + } + if (Date.now() > deadline) throw error; + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20); + } + } + try { + return run(); + } finally { + try { + rmdirSync(lock); + } catch { + // Already removed as stale. + } + } +} + +/** Tokens of one credential that are still valid, oldest first. */ +export function readFeedbackTokens( + scope: string, + now = Date.now() +): StoredFeedbackToken[] { + const { store, pruned } = load(now); + if (pruned) { + try { + withLock(() => { + const current = load(now); + if (current.pruned) save(current.store); + }); + } catch { + // Expired entries are ignored on every read either way. + } + } + return store.entries.filter((entry) => entry.scope === scope); +} + +/** + * Stores call tokens and returns each one's ref, or an empty string for a token the + * cap dropped. A token seen before keeps its ref, and the oldest tokens of the scope + * are dropped past the limit. Storage failures never fail the command. + */ +export function recordFeedbackTokens( + scope: string, + tokens: string[], + now = Date.now() +): string[] { + try { + return withLock(() => { + const { store } = load(now); + const refs = tokens.map((token) => { + const known = store.entries.find( + (entry) => entry.scope === scope && entry.token === token + ); + if (known) return known.ref; + const ref = `c${++store.counter}`; + store.entries.push({ ref, token, scope, at: now }); + return ref; + }); + const mine = store.entries.filter((entry) => entry.scope === scope); + const dropped = new Set(mine.slice(0, -FEEDBACK_TOKEN_LIMIT)); + store.entries = store.entries.filter((entry) => !dropped.has(entry)); + save(store); + const kept = new Set(store.entries.map((entry) => entry.ref)); + return refs.map((ref) => (kept.has(ref) ? ref : '')); + }); + } catch { + return []; + } +} + +export function clearFeedbackTokens( + scope: string, + tokens: string[], + now = Date.now() +): void { + try { + const sent = new Set(tokens); + withLock(() => { + const { store, pruned } = load(now); + const kept = store.entries.filter( + (entry) => entry.scope !== scope || !sent.has(entry.token) + ); + if (!pruned && kept.length === store.entries.length) return; + save({ ...store, entries: kept }); + }); + } catch { + // The tokens expire on their own. + } +}