diff --git a/apps/sim/app/api/cron/workspace-file-search-dispatch/route.ts b/apps/sim/app/api/cron/workspace-file-search-dispatch/route.ts index 2ab25d6ba2f..f245a89906c 100644 --- a/apps/sim/app/api/cron/workspace-file-search-dispatch/route.ts +++ b/apps/sim/app/api/cron/workspace-file-search-dispatch/route.ts @@ -16,8 +16,8 @@ export const GET = withRouteHandler(async (request: NextRequest) => { try { const result = await enqueueWorkspaceFileSearchDispatch() - logger.info('Workspace file search dispatcher accepted', result) - return NextResponse.json({ success: true, triggered: true, ...result }, { status: 202 }) + if (result.triggered) logger.info('Workspace file search dispatcher accepted', result) + return NextResponse.json({ success: true, ...result }, { status: result.triggered ? 202 : 200 }) } catch (error) { logger.error('Workspace file search dispatcher enqueue failed', { error: toError(error).message, diff --git a/apps/sim/app/api/webhooks/outbox/process/route.ts b/apps/sim/app/api/webhooks/outbox/process/route.ts index 6f590e0d33e..e7dfc00255e 100644 --- a/apps/sim/app/api/webhooks/outbox/process/route.ts +++ b/apps/sim/app/api/webhooks/outbox/process/route.ts @@ -20,12 +20,12 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() try { const accepted = await enqueueOutboxProcessor() + if (!accepted.triggered) { + return NextResponse.json({ success: true, requestId, triggered: false }) + } if (accepted.backend === 'trigger-dev') { logger.info('Outbox processor accepted', { jobId: accepted.jobId }) - return NextResponse.json( - { success: true, requestId, triggered: true, ...accepted }, - { status: 202 } - ) + return NextResponse.json({ success: true, requestId, ...accepted }, { status: 202 }) } return NextResponse.json({ success: true, requestId, ...accepted.output }) } catch (error) { diff --git a/apps/sim/lib/core/async-jobs/scheduled-pass.ts b/apps/sim/lib/core/async-jobs/scheduled-pass.ts new file mode 100644 index 00000000000..86ac771ea2a --- /dev/null +++ b/apps/sim/lib/core/async-jobs/scheduled-pass.ts @@ -0,0 +1,73 @@ +import type { TriggerOptions } from '@trigger.dev/sdk' + +/** + * What one cron tick did with a scheduled pass: started none, handed one to Trigger.dev, or + * started one in this process. + */ +export type ScheduledPassResult = + | { triggered: false; backend: null; jobId: null } + | { triggered: true; backend: 'trigger-dev'; jobId: string } + | { triggered: true; backend: 'inline'; jobId: null } + +/** The Trigger.dev run a scheduled pass starts, deduplicated to one per schedule window. */ +interface ScheduledPassTrigger { + taskId: string + /** The idempotency key is `${keyPrefix}:${window}`; defaults to the task id. */ + keyPrefix?: string + /** The schedule window's length: every tick inside one window starts the same run. */ + intervalMs: number + /** The instant whose window keys the run; defaults to the moment the run is triggered. */ + at?: Date + options?: Pick +} + +interface ScheduledPass { + /** Whether this tick owes a pass. The call site owns that policy, including a failed check. */ + due: boolean + /** + * Whether Trigger.dev runs the pass. Read only once the pass is due, so an availability check + * with side effects of its own never runs on a tick that starts nothing. + */ + triggerAvailable: () => boolean + /** Starts the pass in this process without waiting for it. */ + startInline: () => void + trigger: ScheduledPassTrigger +} + +/** + * Triggers the window's run and resolves its id once Trigger.dev durably accepts it. Duplicate + * ticks in one window share the idempotency key, and the key outlives its window so a late + * duplicate is still folded into the run its window started. + */ +export async function triggerScheduledPass(trigger: ScheduledPassTrigger): Promise { + const [{ tasks }, { resolveTriggerRegion }] = await Promise.all([ + import('@trigger.dev/sdk'), + import('@/lib/core/async-jobs/region'), + ]) + const window = Math.floor((trigger.at?.getTime() ?? Date.now()) / trigger.intervalMs) + const handle = await tasks.trigger(trigger.taskId, undefined, { + idempotencyKey: `${trigger.keyPrefix ?? trigger.taskId}:${window}`, + idempotencyKeyTTL: '5m', + ...trigger.options, + region: await resolveTriggerRegion(), + }) + return handle.id +} + +/** + * Runs one cron tick of a scheduled pass: nothing when no pass is due, the window's Trigger.dev + * run when Trigger.dev is available, and otherwise a pass started in this process. The result + * returns once the pass is accepted, never after it finishes. + */ +export async function startScheduledPass(pass: ScheduledPass): Promise { + if (!pass.due) return { triggered: false, backend: null, jobId: null } + if (!pass.triggerAvailable()) { + pass.startInline() + return { triggered: true, backend: 'inline', jobId: null } + } + return { + triggered: true, + backend: 'trigger-dev', + jobId: await triggerScheduledPass(pass.trigger), + } +} diff --git a/apps/sim/lib/core/outbox/constants.ts b/apps/sim/lib/core/outbox/constants.ts index c19580a9832..4bd915bfc0a 100644 --- a/apps/sim/lib/core/outbox/constants.ts +++ b/apps/sim/lib/core/outbox/constants.ts @@ -6,3 +6,5 @@ export const OUTBOX_PROCESSOR_INTERVAL_MS = 60_000 export const OUTBOX_PROCESSOR_CONCURRENCY = Math.ceil( (OUTBOX_PROCESSOR_MAX_DURATION_SECONDS * 1000) / OUTBOX_PROCESSOR_INTERVAL_MS ) +/** Every Nth scheduled tick runs the processor's recovery, reaping and pruning even with nothing due. */ +export const OUTBOX_MAINTENANCE_EVERY_INTERVALS = 5 diff --git a/apps/sim/lib/core/outbox/enqueue.test.ts b/apps/sim/lib/core/outbox/enqueue.test.ts index 493c1537ff7..3cda982d5eb 100644 --- a/apps/sim/lib/core/outbox/enqueue.test.ts +++ b/apps/sim/lib/core/outbox/enqueue.test.ts @@ -1,5 +1,10 @@ import { asyncJobsRegionMock } from '@sim/testing/mocks/async-jobs-region.mock' import { setEnvFlags } from '@sim/testing/mocks/env-flags.mock' +import { outboxServiceMock, outboxServiceMockFns } from '@sim/testing/mocks/outbox-service.mock' +import { + createIdempotentTasksTrigger, + triggerSdkMockFns, +} from '@sim/testing/mocks/trigger-sdk.mock' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const hoisted = vi.hoisted(() => ({ @@ -7,29 +12,67 @@ const hoisted = vi.hoisted(() => ({ })) vi.mock('@/lib/core/async-jobs/region', () => asyncJobsRegionMock) vi.mock('@/lib/core/outbox/processor', () => ({ runOutboxProcessor: hoisted.processor })) +vi.mock('@/lib/core/outbox/service', () => outboxServiceMock) -import { tasks } from '@trigger.dev/sdk' import { enqueueOutboxProcessor } from '@/lib/core/outbox/enqueue' -const mocks = { ...hoisted, trigger: vi.mocked(tasks.trigger) } +const mocks = { + ...hoisted, + trigger: triggerSdkMockFns.mockTasksTrigger, + hasDueWork: outboxServiceMockFns.mockHasDueOutboxWork, +} + +/** 12:34 is not a maintenance minute (754 % 5 = 4); 12:35 is. */ +const IDLE_MINUTE = new Date('2026-09-16T12:34:45Z') +const MAINTENANCE_MINUTE = new Date('2026-09-16T12:35:10Z') + +const INLINE_OUTPUT = { + result: { processed: 0, retried: 0, deadLettered: 0, leaseLost: 0, reaped: 0 }, + recoveredDocuments: 0, + reapedBackgroundWork: 0, +} + +/** Each backend's own proof that it, and only it, ran the processor. */ +const BACKENDS = [ + { + name: 'Trigger.dev', + isTriggerDevEnabled: true, + started: { triggered: true, backend: 'trigger-dev', jobId: 'run-1' }, + }, + { + name: 'self-hosted inline', + isTriggerDevEnabled: false, + started: { triggered: true, backend: 'inline', jobId: null, output: INLINE_OUTPUT }, + }, +] as const describe('outbox processor enqueue', () => { beforeEach(() => { vi.useFakeTimers() - vi.setSystemTime(new Date('2026-09-16T12:34:45Z')) + vi.setSystemTime(IDLE_MINUTE) setEnvFlags({ isTriggerDevEnabled: true }) - mocks.trigger.mockResolvedValue({ id: 'run-1' }) + mocks.trigger.mockImplementation(createIdempotentTasksTrigger()) + mocks.processor.mockReset() + mocks.processor.mockResolvedValue(INLINE_OUTPUT) + mocks.hasDueWork.mockReset() + mocks.hasDueWork.mockResolvedValue(true) }) afterEach(() => vi.useRealTimers()) - it('deduplicates duplicate ticks while allowing the next minute to drain more work', async () => { - await enqueueOutboxProcessor() - await enqueueOutboxProcessor() - vi.advanceTimersByTime(60_000) - await enqueueOutboxProcessor() - const keys = mocks.trigger.mock.calls.map((call) => call[2].idempotencyKey) - expect(keys[0]).toBe(keys[1]) - expect(keys[2]).not.toBe(keys[0]) + it('folds ticks into the run of the minute their work check was made in', async () => { + const first = await enqueueOutboxProcessor() + const duplicate = await enqueueOutboxProcessor() + mocks.hasDueWork.mockImplementationOnce(async () => { + vi.advanceTimersByTime(60_000) + return true + }) + const checkedAcrossTheBoundary = await enqueueOutboxProcessor() + const nextMinute = await enqueueOutboxProcessor() + + expect(first.jobId).toBe('run-1') + expect(duplicate.jobId).toBe('run-1') + expect(checkedAcrossTheBoundary.jobId).toBe('run-1') + expect(nextMinute.jobId).toBe('run-2') }) it('fails closed on an enqueue error without starting concurrent inline work', async () => { @@ -46,7 +89,44 @@ describe('outbox processor enqueue', () => { reapedBackgroundWork: 1, } mocks.processor.mockResolvedValueOnce(output) - await expect(enqueueOutboxProcessor()).resolves.toEqual({ backend: 'inline', output }) + await expect(enqueueOutboxProcessor()).resolves.toEqual({ + triggered: true, + backend: 'inline', + jobId: null, + output, + }) expect(mocks.trigger).not.toHaveBeenCalled() }) + + it.each(BACKENDS)( + 'starts no $name processor on an idle queue outside the maintenance window', + async ({ isTriggerDevEnabled }) => { + setEnvFlags({ isTriggerDevEnabled }) + mocks.hasDueWork.mockResolvedValue(false) + await expect(enqueueOutboxProcessor()).resolves.toEqual({ + triggered: false, + backend: null, + jobId: null, + }) + } + ) + + it.each(BACKENDS)( + 'starts the $name processor on an idle queue in the maintenance window', + async ({ isTriggerDevEnabled, started }) => { + setEnvFlags({ isTriggerDevEnabled }) + vi.setSystemTime(MAINTENANCE_MINUTE) + mocks.hasDueWork.mockResolvedValue(false) + await expect(enqueueOutboxProcessor()).resolves.toEqual(started) + } + ) + + it.each(BACKENDS)( + 'starts the $name processor when the work check fails', + async ({ isTriggerDevEnabled, started }) => { + setEnvFlags({ isTriggerDevEnabled }) + mocks.hasDueWork.mockRejectedValue(new Error('connection refused')) + await expect(enqueueOutboxProcessor()).resolves.toEqual(started) + } + ) }) diff --git a/apps/sim/lib/core/outbox/enqueue.ts b/apps/sim/lib/core/outbox/enqueue.ts index 95f2bf9e03d..42213fc5c89 100644 --- a/apps/sim/lib/core/outbox/enqueue.ts +++ b/apps/sim/lib/core/outbox/enqueue.ts @@ -1,32 +1,65 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { + type ScheduledPassResult, + triggerScheduledPass, +} from '@/lib/core/async-jobs/scheduled-pass' import { isTriggerDevEnabled } from '@/lib/core/config/env-flags' import { + OUTBOX_MAINTENANCE_EVERY_INTERVALS, OUTBOX_PROCESSOR_INTERVAL_MS, OUTBOX_PROCESSOR_MAX_DURATION_SECONDS, } from '@/lib/core/outbox/constants' import type { OutboxProcessorResult } from '@/lib/core/outbox/processor' -import type { processOutboxTask } from '@/background/process-outbox' +import { hasDueOutboxWork } from '@/lib/core/outbox/service' + +const logger = createLogger('OutboxProcessorEnqueue') +/** A self-hosted inline run finishes before the cron request returns, so it carries its output. */ type OutboxProcessorEnqueueResult = - | { backend: 'trigger-dev'; jobId: string } - | { backend: 'inline'; output: OutboxProcessorResult } + | Exclude + | { triggered: true; backend: 'inline'; jobId: null; output: OutboxProcessorResult } -/** The database owns delivery state; the cron request waits only for durable worker acceptance. */ +/** + * Every tick in the maintenance window runs, so document recovery, background-work reaping and + * pruning keep their cadence on an idle queue. Other ticks run only when events are due or a lease + * is stale; a gate that cannot answer runs anyway, since billing, seat sync, invitations and + * document dispatch all ride the outbox. + */ +async function shouldRunOutboxProcessor(now: Date, scheduleWindow: number): Promise { + if (scheduleWindow % OUTBOX_MAINTENANCE_EVERY_INTERVALS === 0) return true + try { + return await hasDueOutboxWork(now) + } catch (error) { + logger.warn('Outbox work check failed; running the processor anyway', { + error: getErrorMessage(error), + }) + return true + } +} + +/** + * The database owns delivery state; the cron request waits only for durable worker acceptance. + * The self-hosted branch processes synchronously and returns the output the route reports, which + * is why it stays here instead of in the scheduled pass's detached inline start. + */ export async function enqueueOutboxProcessor(): Promise { + const now = new Date() + const scheduleWindow = Math.floor(now.getTime() / OUTBOX_PROCESSOR_INTERVAL_MS) + if (!(await shouldRunOutboxProcessor(now, scheduleWindow))) { + return { triggered: false, backend: null, jobId: null } + } + if (!isTriggerDevEnabled) { const { runOutboxProcessor } = await import('@/lib/core/outbox/processor') - return { backend: 'inline', output: await runOutboxProcessor() } + return { triggered: true, backend: 'inline', jobId: null, output: await runOutboxProcessor() } } - const [{ tasks }, { resolveTriggerRegion }] = await Promise.all([ - import('@trigger.dev/sdk'), - import('@/lib/core/async-jobs/region'), - ]) - const scheduleWindow = Math.floor(Date.now() / OUTBOX_PROCESSOR_INTERVAL_MS) - const handle = await tasks.trigger('process-outbox', undefined, { - idempotencyKey: `process-outbox:${scheduleWindow}`, - idempotencyKeyTTL: '5m', - maxDuration: OUTBOX_PROCESSOR_MAX_DURATION_SECONDS, - region: await resolveTriggerRegion(), + const jobId = await triggerScheduledPass({ + taskId: 'process-outbox', + intervalMs: OUTBOX_PROCESSOR_INTERVAL_MS, + at: now, + options: { maxDuration: OUTBOX_PROCESSOR_MAX_DURATION_SECONDS }, }) - return { backend: 'trigger-dev', jobId: handle.id } + return { triggered: true, backend: 'trigger-dev', jobId } } diff --git a/apps/sim/lib/core/outbox/queries.ts b/apps/sim/lib/core/outbox/queries.ts index 6176e169468..f009ccc337b 100644 --- a/apps/sim/lib/core/outbox/queries.ts +++ b/apps/sim/lib/core/outbox/queries.ts @@ -1,7 +1,18 @@ +import { db } from '@sim/db' import { outboxEvent } from '@sim/db/schema' -import { sql } from 'drizzle-orm' +import { and, eq, exists, lte, or, type SQL, sql } from 'drizzle-orm' const MAX_READY_EVENT_TYPES = 128 +/** How long a `processing` lease may go without a terminal write before the reaper reclaims it. */ +export const STUCK_PROCESSING_THRESHOLD_MS = 10 * 60 * 1000 + +/** A `processing` row whose lease the reaper may reclaim at `now`. */ +export function isStuckProcessing(now: Date): SQL | undefined { + return and( + eq(outboxEvent.status, 'processing'), + lte(outboxEvent.lockedAt, new Date(now.getTime() - STUCK_PROCESSING_THRESHOLD_MS)) + ) +} /** * Walks the pending index one type at a time, reading only its earliest availability. @@ -42,3 +53,18 @@ export function readyEventTypesQuery(now: Date) { LIMIT ${MAX_READY_EVENT_TYPES} ` } + +/** + * One row, `due`: whether `processOutboxEvents` would act at `now`. The pending leg is the + * claim phase's own discovery walk, so it reads only each type's head and cannot be planned as a + * scan of future or completed rows; the lease leg is the reaper's predicate, over the few rows in + * `processing`. + */ +export function dueOutboxWorkQuery(now: Date) { + const staleLease = db + .select({ id: outboxEvent.id }) + .from(outboxEvent) + .where(isStuckProcessing(now)) + .limit(1) + return sql`SELECT ${or(sql`EXISTS (${readyEventTypesQuery(now)})`, exists(staleLease))} AS due` +} diff --git a/apps/sim/lib/core/outbox/retention.ts b/apps/sim/lib/core/outbox/retention.ts index b9538baf95f..9244ec36d82 100644 --- a/apps/sim/lib/core/outbox/retention.ts +++ b/apps/sim/lib/core/outbox/retention.ts @@ -10,9 +10,8 @@ const logger = createLogger('OutboxRetention') /** How long a completed event stays readable for operators after it was enqueued. */ export const COMPLETED_OUTBOX_RETENTION_MS = 7 * 24 * 60 * 60_000 /** - * Rows deleted per type per run. The processor runs once a minute, so each type drains at most - * 1,000 rows × 1,440 runs = 1.44M rows a day: a steady trickle whose WAL and dead tuples - * autovacuum absorbs, yet five times what recovery can enqueue (200 per run × 1,440 runs). + * Most rows deleted per type per run. Recovery enqueues only inside a run, at most 200 per run, so + * a 1,000-row prune keeps five times its pace however often the processor runs. */ export const OUTBOX_PRUNE_BATCH_SIZE = 1_000 diff --git a/apps/sim/lib/core/outbox/service.integration.ts b/apps/sim/lib/core/outbox/service.integration.ts index c2d9e5857f5..589c41c5659 100644 --- a/apps/sim/lib/core/outbox/service.integration.ts +++ b/apps/sim/lib/core/outbox/service.integration.ts @@ -19,8 +19,13 @@ vi.mock('@sim/db', () => ({ }, })) -import { readyEventTypesQuery } from '@/lib/core/outbox/queries' import { + dueOutboxWorkQuery, + readyEventTypesQuery, + STUCK_PROCESSING_THRESHOLD_MS, +} from '@/lib/core/outbox/queries' +import { + hasDueOutboxWork, type OutboxHandler, processOutboxEvents, withOutboxHandlerTimeout, @@ -386,4 +391,62 @@ describe('outbox scheduling in PostgreSQL', () => { const second = await processOutboxEvents({}, { batchSize: 0 }) expect(second.reaped).toBe(5) }) + + describe('due-work gate', () => { + const now = new Date('2026-09-16T12:34:00.000Z') + + async function insertRow( + status: 'pending' | 'processing' | 'completed' | 'dead_letter', + times: { availableAt?: Date; lockedAt?: Date | null } + ) { + eventTypes.add('test.outbox.gate') + await db.insert(outboxEvent).values({ + id: generateId(), + eventType: 'test.outbox.gate', + payload: {}, + status, + createdAt: new Date(now.getTime() - 60 * 60_000), + availableAt: times.availableAt ?? new Date(now.getTime() - 60 * 60_000), + lockedAt: times.lockedAt ?? null, + }) + } + + it('reports no work for future, freshly leased, completed and dead-lettered rows', async () => { + expect(await hasDueOutboxWork(now)).toBe(false) + await insertRow('pending', { availableAt: new Date(now.getTime() + 1) }) + await insertRow('processing', { + lockedAt: new Date(now.getTime() - STUCK_PROCESSING_THRESHOLD_MS + 1), + }) + await insertRow('completed', {}) + await insertRow('dead_letter', {}) + expect(await hasDueOutboxWork(now)).toBe(false) + }) + + it('reports a pending event due exactly now, as the claim phase would take it', async () => { + await insertRow('pending', { availableAt: now }) + expect(await hasDueOutboxWork(now)).toBe(true) + }) + + it('reports a lease exactly at the stale threshold, as the reaper would reclaim it', async () => { + await insertRow('processing', { + lockedAt: new Date(now.getTime() - STUCK_PROCESSING_THRESHOLD_MS), + }) + expect(await hasDueOutboxWork(now)).toBe(true) + }) + + it('probes both legs through indexes beside a large completed and future backlog', async () => { + await seedBacklog('test.outbox.gate', 50_000, 'completed') + await seedBacklog('test.outbox.gate', 50_000, 'pending', new Date(Date.now() + 60 * 60_000)) + await connection`VACUUM (ANALYZE) outbox_event` + const gateNow = new Date() + expect(await hasDueOutboxWork(gateNow)).toBe(false) + + const plans = await db.execute<{ 'QUERY PLAN': { Plan: QueryPlan }[] }>(sql` + EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${dueOutboxWorkQuery(gateNow)} + `) + const plan = plans[0]['QUERY PLAN'][0].Plan + expect(planNodes(plan).some((node) => node['Node Type'] === 'Seq Scan')).toBe(false) + expect(plan['Shared Hit Blocks'] + plan['Shared Read Blocks']).toBeLessThan(100) + }, 60_000) + }) }) diff --git a/apps/sim/lib/core/outbox/service.ts b/apps/sim/lib/core/outbox/service.ts index 54ce6e779f4..472aa524232 100644 --- a/apps/sim/lib/core/outbox/service.ts +++ b/apps/sim/lib/core/outbox/service.ts @@ -5,7 +5,12 @@ import { describeError, toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { truncate } from '@sim/utils/string' import { and, asc, desc, eq, inArray, lte, sql } from 'drizzle-orm' -import { readyEventTypesQuery } from '@/lib/core/outbox/queries' +import { + dueOutboxWorkQuery, + isStuckProcessing, + readyEventTypesQuery, + STUCK_PROCESSING_THRESHOLD_MS, +} from '@/lib/core/outbox/queries' const logger = createLogger('OutboxService') @@ -24,7 +29,6 @@ const MAX_REAPED_EVENTS = 1_000 function toPersistedHandlerError(error: unknown): string { return truncate(toError(error).message.split(/\nparams: /)[0], MAX_PERSISTED_ERROR_LENGTH) } -const STUCK_PROCESSING_THRESHOLD_MS = 10 * 60 * 1000 // 10 minutes const MAX_BACKOFF_MS = 60 * 60 * 1000 // 1 hour const BASE_BACKOFF_MS = 1000 // 1 second, doubled per attempt /** Ordinary handlers keep a short window; longer handlers explicitly opt in below the stale-lease limit. */ @@ -542,6 +546,15 @@ export async function processOutboxEventById( return runHandler(event, handlers) } +/** + * Whether {@link processOutboxEvents} would find anything at `now`: a pending event the claim + * phase may take, or a stale lease the reaper may reclaim. + */ +export async function hasDueOutboxWork(now: Date): Promise { + const [row] = await db.execute<{ due: boolean }>(dueOutboxWorkQuery(now)) + return row?.due === true +} + /** * Reaper: move `processing` rows whose worker died (stale `lockedAt`) * back to `pending` so another worker can pick them up. Without this, @@ -549,11 +562,10 @@ export async function processOutboxEventById( * the row in `processing`. */ async function reapStuckProcessingRows(): Promise { - const stuckBefore = new Date(Date.now() - STUCK_PROCESSING_THRESHOLD_MS) const stuckRows = db .select({ id: outboxEvent.id }) .from(outboxEvent) - .where(and(eq(outboxEvent.status, 'processing'), lte(outboxEvent.lockedAt, stuckBefore))) + .where(isStuckProcessing(new Date())) .orderBy(asc(outboxEvent.lockedAt), asc(outboxEvent.id)) .limit(MAX_REAPED_EVENTS) .for('update', { skipLocked: true }) diff --git a/apps/sim/lib/knowledge/projection/enqueue.ts b/apps/sim/lib/knowledge/projection/enqueue.ts index 69592a29e2a..08c39dec0ed 100644 --- a/apps/sim/lib/knowledge/projection/enqueue.ts +++ b/apps/sim/lib/knowledge/projection/enqueue.ts @@ -6,6 +6,7 @@ import { } from '@sim/db/knowledge-projection' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { type ScheduledPassResult, startScheduledPass } from '@/lib/core/async-jobs/scheduled-pass' import { isTriggerAvailable } from '@/lib/core/config/trigger-availability' const logger = createLogger('KnowledgeProjectionEnqueue') @@ -57,13 +58,6 @@ function runInline(): void { })() } -export interface KnowledgeProjectionSweepResult { - /** Whether a pass was started; the sweep starts none when nothing is marked. */ - triggered: boolean - backend: 'trigger-dev' | 'inline' | null - jobId: string | null -} - /** * The knowledge projector's only trigger: one pass per window while marks need one, so marks are * settled within about a minute of their write, a document a pass gave up is retried by the next, @@ -72,7 +66,7 @@ export interface KnowledgeProjectionSweepResult { * Search marks and permission-only marks are released without copying their rows. Only deferred * ordinary-KB content admits a repair pass. */ -export async function enqueueKnowledgeProjectionSweep(): Promise { +export async function enqueueKnowledgeProjectionSweep(): Promise { let release = { drained: false, empty: false } try { release = await releaseSettledMarks(db.$client, Date.now() + MARK_RELEASE_BUDGET_MS) @@ -80,23 +74,15 @@ export async function enqueueKnowledgeProjectionSweep(): Promise ({ indexWorkspaceFileForSearch: vi.fn(), markWorkspaceFileSearchIndexFailed: vi.fn(), })) -vi.mock('@/lib/workspace-files/search/index-state', () => ({ - cleanupFileSearchBuilds: vi.fn().mockResolvedValue(0), -})) vi.mock('@trigger.dev/sdk', () => ({ tasks: { batchTrigger: mocks.batchTrigger } })) vi.mock('@/lib/core/config/env-flags', () => ({ isTriggerDevEnabled: true })) vi.mock('@/lib/core/async-jobs/region', () => ({ resolveTriggerRegion: async () => 'us-east-1' })) @@ -30,12 +27,27 @@ import { FILE_SEARCH_BACKFILL_PAGE_SIZE, FILE_SEARCH_DISPATCH_HANDOFF_MS, FILE_SEARCH_INDEX_STALE_DISPATCH_MS, + FILE_SEARCH_RECONCILE_INTERVAL_MS, } from '@/lib/workspace-files/search/constants' import { dispatchWorkspaceFileSearchIndexJobs, + hasWorkspaceFileSearchDispatchWork, prepareWorkspaceFileSearchDispatch, } from '@/lib/workspace-files/search/dispatcher' +interface QueryPlan { + 'Node Type': string + 'Relation Name'?: string + 'Index Name'?: string + 'Shared Hit Blocks': number + 'Shared Read Blocks': number + Plans?: QueryPlan[] +} + +function planNodes(plan: QueryPlan): QueryPlan[] { + return [plan, ...(plan.Plans ?? []).flatMap(planNodes)] +} + describe('workspace file search dispatch PostgreSQL deadlines', () => { const schemaName = `dispatch_test_${generateId().replaceAll('-', '')}` const databaseUrl = readTestDatabaseUrl() @@ -90,11 +102,12 @@ describe('workspace file search dispatch PostgreSQL deadlines', () => { ON workspace_file_search_revision (workspace_id, updated_at, file_id, source_content_updated_at) WHERE status = 'pending' AND dispatched_at IS NULL` - await connection`CREATE INDEX ON workspace_file_search_revision (workspace_id, dispatched_at) + await connection`CREATE INDEX workspace_file_search_revision_active_idx + ON workspace_file_search_revision (dispatched_at, workspace_id) WHERE status = 'pending' AND dispatched_at IS NOT NULL` - await connection`INSERT INTO workspace_file_search_backfill (id, updated_at) - VALUES ('workspace-file-search-chunks-v2', '2026-09-16 00:00:00')` await connection`CREATE TABLE workspace_file_search_build (id text PRIMARY KEY, expires_at timestamp)` + await connection`CREATE INDEX workspace_file_search_build_cleanup_idx + ON workspace_file_search_build (expires_at, id) WHERE expires_at IS NOT NULL` await connection`CREATE TABLE workspace_file_search_chunk (build_id text NOT NULL, ordinal integer NOT NULL, PRIMARY KEY(build_id, ordinal))` database.current = drizzle(connection) }) @@ -102,9 +115,11 @@ describe('workspace file search dispatch PostgreSQL deadlines', () => { beforeEach(async () => { mocks.batchTrigger.mockReset() await connection`DROP TRIGGER IF EXISTS slow_backfill ON workspace_file_search_backfill` - await connection`TRUNCATE workspace_files, workspace_file_search_revision, workspace_file_search_dispatch_queue` - await connection`UPDATE workspace_file_search_backfill - SET updated_at = '2026-09-16 00:00:00', completed_at = NULL, + await connection`TRUNCATE workspace_files, workspace_file_search_revision, + workspace_file_search_dispatch_queue, workspace_file_search_build` + await connection`INSERT INTO workspace_file_search_backfill (id, updated_at) + VALUES ('workspace-file-search-chunks-v2', '2026-09-16 00:00:00') + ON CONFLICT (id) DO UPDATE SET updated_at = EXCLUDED.updated_at, completed_at = NULL, after_workspace_id = NULL, after_file_id = NULL` }) @@ -544,4 +559,117 @@ describe('workspace file search dispatch PostgreSQL deadlines', () => { { lock_timeout: '0', statement_timeout: '0', transaction_timeout: '0' }, ]) }) + + describe('dispatch work check', () => { + const reconcileSeconds = FILE_SEARCH_RECONCILE_INTERVAL_MS / 1000 + const staleSeconds = FILE_SEARCH_INDEX_STALE_DISPATCH_MS / 1000 + + /** + * Everything a deployment between file changes holds: a reconciled backfill, a published + * revision, a claim whose run is under way, a live build, and a build still inside its lease. + */ + async function seedIdleDeployment() { + await connection`UPDATE workspace_file_search_backfill + SET completed_at = now() - make_interval(secs => ${reconcileSeconds - 60})` + await connection`INSERT INTO workspace_files (id, workspace_id, context, content_updated_at) + VALUES ('ready-file', 'workspace-1', 'workspace', '2026-09-16'), + ('running-file', 'workspace-1', 'workspace', '2026-09-16')` + await connection`INSERT INTO workspace_file_search_revision + (file_id, workspace_id, source_content_updated_at, status, dispatched_at, handoff_expires_at) + VALUES ('ready-file', 'workspace-1', '2026-09-16', 'ready', NULL, NULL), + ('running-file', 'workspace-1', '2026-09-16', 'pending', + now() - make_interval(secs => ${staleSeconds - 60}), clock_timestamp() + interval '1 minute')` + await connection`INSERT INTO workspace_file_search_build (id, expires_at) + VALUES ('published-build', NULL), ('leased-build', now() + interval '1 minute')` + } + + it('reports no work for an idle deployment, on which a dispatch pass does nothing', async () => { + await seedIdleDeployment() + + await expect(hasWorkspaceFileSearchDispatchWork(new Date())).resolves.toBe(false) + await expect(prepareWorkspaceFileSearchDispatch()).resolves.toMatchObject({ + payloads: [], + backfilledFiles: 0, + reapedClaims: 0, + }) + }) + + it.each([ + { + work: 'a backfill pass that never completed', + seed: () => connection`UPDATE workspace_file_search_backfill SET completed_at = NULL`, + }, + { + work: 'a missing backfill cursor', + seed: () => connection`DELETE FROM workspace_file_search_backfill`, + }, + { + work: 'a backfill reconcile that is due', + seed: () => connection`UPDATE workspace_file_search_backfill + SET completed_at = now() - make_interval(secs => ${reconcileSeconds})`, + }, + { + work: 'a queued workspace', + seed: () => connection`INSERT INTO workspace_file_search_dispatch_queue + (workspace_id, enqueued_at, updated_at) VALUES ('workspace-1', now(), now())`, + }, + { + work: 'an expired build', + seed: () => connection`UPDATE workspace_file_search_build SET expires_at = now() + WHERE id = 'leased-build'`, + }, + { + work: 'a claim past the stale-dispatch window', + seed: () => connection`UPDATE workspace_file_search_revision + SET dispatched_at = now() - make_interval(secs => ${staleSeconds + 60}) + WHERE file_id = 'running-file'`, + }, + { + work: 'a claim past its handoff deadline', + seed: () => connection`UPDATE workspace_file_search_revision + SET handoff_expires_at = clock_timestamp() - interval '1 millisecond' + WHERE file_id = 'running-file'`, + }, + ])('reports work for $work', async ({ seed }) => { + await seedIdleDeployment() + await seed() + + await expect(hasWorkspaceFileSearchDispatchWork(new Date())).resolves.toBe(true) + }) + + it('probes builds and claims through their indexes beside a large settled backlog', async () => { + await seedIdleDeployment() + await connection`INSERT INTO workspace_file_search_revision + (file_id, workspace_id, source_content_updated_at, status) + SELECT 'settled-' || n, 'workspace-' || (n % 50), '2026-09-16', 'ready' + FROM generate_series(1, 100000) n` + await connection`INSERT INTO workspace_file_search_build (id, expires_at) + SELECT 'published-' || n, NULL FROM generate_series(1, 100000) n` + await connection`VACUUM (ANALYZE) workspace_file_search_revision, workspace_file_search_build` + statements.length = 0 + await expect(hasWorkspaceFileSearchDispatchWork(new Date())).resolves.toBe(false) + + const probe = statements.find((statement) => statement.query.includes('"hasWork"')) + expect(probe).toBeDefined() + const [explained] = await connection.unsafe( + `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${probe?.query}`, + probe?.params as never[] + ) + const plan = (explained['QUERY PLAN'] as { Plan: QueryPlan }[])[0].Plan + const nodes = planNodes(plan) + const indexes = nodes.map((node) => node['Index Name']) + + expect(indexes).toContain('workspace_file_search_build_cleanup_idx') + expect(indexes).toContain('workspace_file_search_revision_active_idx') + expect( + nodes.filter( + (node) => + node['Node Type'] === 'Seq Scan' && + node['Relation Name'] !== 'workspace_file_search_dispatch_queue' + ) + ).toEqual([]) + /** Buffer work, unlike wall-clock time, catches a backlog scan even on a warm local database. */ + expect(plan['Shared Hit Blocks'] + plan['Shared Read Blocks']).toBeLessThan(100) + }, 60_000) + }) }) diff --git a/apps/sim/lib/workspace-files/search/dispatcher.ts b/apps/sim/lib/workspace-files/search/dispatcher.ts index 377380f0ed0..ddfbb9a991d 100644 --- a/apps/sim/lib/workspace-files/search/dispatcher.ts +++ b/apps/sim/lib/workspace-files/search/dispatcher.ts @@ -1,6 +1,7 @@ import { db } from '@sim/db' import { workspaceFileSearchBackfill, + workspaceFileSearchBuild, workspaceFileSearchDispatchQueue, workspaceFileSearchRevision, workspaceFiles, @@ -48,7 +49,10 @@ import { FILE_SEARCH_INDEX_WORKSPACE_OUTSTANDING, FILE_SEARCH_RECONCILE_INTERVAL_MS, } from '@/lib/workspace-files/search/constants' -import { cleanupFileSearchBuilds } from '@/lib/workspace-files/search/index-state' +import { + cleanupFileSearchBuilds, + fileSearchBuildExpired, +} from '@/lib/workspace-files/search/index-state' import { indexWorkspaceFileForSearch, markWorkspaceFileSearchIndexFailed, @@ -166,6 +170,32 @@ async function enqueueWorkspaces( }) } +/** + * Whether the backfill walk owes a page: the cursor has never completed a pass, or its last + * complete pass is at least {@link FILE_SEARCH_RECONCILE_INTERVAL_MS} old. + */ +function isBackfillPageDue(completedAt: Date | null, now: Date): boolean { + return !completedAt || now.getTime() - completedAt.getTime() >= FILE_SEARCH_RECONCILE_INTERVAL_MS +} + +/** + * A claim {@link reapStaleClaims} releases: its handoff deadline has passed (in PostgreSQL time), or + * it has outlasted the stale-dispatch window. Served by `workspace_file_search_revision_active_idx`. + */ +function staleClaim(now: Date): SQL | undefined { + return and( + eq(workspaceFileSearchRevision.status, 'pending'), + isNotNull(workspaceFileSearchRevision.dispatchedAt), + or( + lt( + workspaceFileSearchRevision.dispatchedAt, + new Date(now.getTime() - FILE_SEARCH_INDEX_STALE_DISPATCH_MS) + ), + lte(workspaceFileSearchRevision.handoffExpiresAt, sql`clock_timestamp()`) + ) + ) +} + /** * Seeds one page of the backfill that walks every live workspace file into the revision table. * @@ -192,12 +222,7 @@ async function seedBackfillPage(tx: DbTransaction, now: Date): Promise { .where(eq(workspaceFileSearchBackfill.id, BACKFILL_CURSOR_ID)) .for('update') .limit(1) - if ( - !cursor || - (cursor.completedAt && - now.getTime() - cursor.completedAt.getTime() < FILE_SEARCH_RECONCILE_INTERVAL_MS) - ) - return 0 + if (!cursor || !isBackfillPageDue(cursor.completedAt, now)) return 0 const afterWorkspaceId = cursor.completedAt ? null : cursor.afterWorkspaceId const afterFileId = cursor.completedAt ? null : cursor.afterFileId @@ -271,7 +296,6 @@ async function reapStaleClaims( tx: DbTransaction, now: Date ): Promise<{ reaped: number; abandoned: number }> { - const staleBefore = new Date(now.getTime() - FILE_SEARCH_INDEX_STALE_DISPATCH_MS) const rows = await tx .select({ workspaceId: workspaceFileSearchRevision.workspaceId, @@ -291,16 +315,7 @@ async function reapStaleClaims( eq(workspaceFiles.contentUpdatedAt, workspaceFileSearchRevision.sourceContentUpdatedAt) ) ) - .where( - and( - eq(workspaceFileSearchRevision.status, 'pending'), - isNotNull(workspaceFileSearchRevision.dispatchedAt), - or( - lt(workspaceFileSearchRevision.dispatchedAt, staleBefore), - lte(workspaceFileSearchRevision.handoffExpiresAt, sql`clock_timestamp()`) - ) - ) - ) + .where(staleClaim(now)) .orderBy(asc(workspaceFileSearchRevision.dispatchedAt), asc(workspaceFileSearchRevision.fileId)) .limit(FILE_SEARCH_INDEX_STALE_REAP_LIMIT) .for('update', { of: workspaceFileSearchRevision, skipLocked: true }) @@ -449,6 +464,42 @@ async function claimQueuedWorkspaceJobs( })) } +/** + * Whether a dispatch pass at `now` has anything to do, so the per-minute cron can skip starting one + * on an idle deployment. Mirrors each phase of {@link dispatchWorkspaceFileSearchIndexJobs} with the + * same predicates: a backfill page is due, a build has expired for cleanup, a claim is stale for the + * reaper, or a workspace is queued for claiming. File writes, file deletes, and released claims + * all land in one of these through the `workspace_file_search_mark_pending` trigger or the + * dispatcher's own writes. + */ +export async function hasWorkspaceFileSearchDispatchWork(now: Date): Promise { + const [cursor] = await db + .select({ completedAt: workspaceFileSearchBackfill.completedAt }) + .from(workspaceFileSearchBackfill) + .where(eq(workspaceFileSearchBackfill.id, BACKFILL_CURSOR_ID)) + .limit(1) + if (!cursor || isBackfillPageDue(cursor.completedAt, now)) return true + + const queuedWorkspace = db + .select({ workspaceId: workspaceFileSearchDispatchQueue.workspaceId }) + .from(workspaceFileSearchDispatchQueue) + .limit(1) + const expiredBuild = db + .select({ id: workspaceFileSearchBuild.id }) + .from(workspaceFileSearchBuild) + .where(fileSearchBuildExpired) + .limit(1) + const staleClaimRow = db + .select({ fileId: workspaceFileSearchRevision.fileId }) + .from(workspaceFileSearchRevision) + .where(staleClaim(now)) + .limit(1) + const [probe] = await db.execute<{ hasWork: boolean }>( + sql`SELECT ${or(exists(queuedWorkspace), exists(expiredBuild), exists(staleClaimRow))} AS "hasWork"` + ) + return probe?.hasWork === true +} + export async function prepareWorkspaceFileSearchDispatch( maxOutstanding = FILE_SEARCH_INDEX_MAX_OUTSTANDING ): Promise { diff --git a/apps/sim/lib/workspace-files/search/enqueue-dispatch.test.ts b/apps/sim/lib/workspace-files/search/enqueue-dispatch.test.ts index b89eb64f93d..ca997b15e4d 100644 --- a/apps/sim/lib/workspace-files/search/enqueue-dispatch.test.ts +++ b/apps/sim/lib/workspace-files/search/enqueue-dispatch.test.ts @@ -1,58 +1,87 @@ -import { - asyncJobsRegionMock, - asyncJobsRegionMockFns, -} from '@sim/testing/mocks/async-jobs-region.mock' -import { backgroundTaskMock, backgroundTaskMockFns } from '@sim/testing/mocks/background-task.mock' +import { asyncJobsRegionMock } from '@sim/testing/mocks/async-jobs-region.mock' +import { backgroundTaskMock } from '@sim/testing/mocks/background-task.mock' import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks/env-flags.mock' +import { + createIdempotentTasksTrigger, + triggerSdkMockFns, +} from '@sim/testing/mocks/trigger-sdk.mock' import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ - dispatch: vi.fn(), + hasWork: vi.fn(), })) -vi.mock('@/background/workspace-file-search-dispatch', () => ({ - workspaceFileSearchDispatchTask: {}, -})) vi.mock('@/lib/core/async-jobs/region', () => asyncJobsRegionMock) vi.mock('@/lib/core/utils/background', () => backgroundTaskMock) vi.mock('@/lib/workspace-files/search/dispatcher', () => ({ - dispatchWorkspaceFileSearchIndexJobs: mocks.dispatch, + dispatchWorkspaceFileSearchIndexJobs: vi.fn(), + hasWorkspaceFileSearchDispatchWork: mocks.hasWork, })) -import { tasks } from '@trigger.dev/sdk' import { enqueueWorkspaceFileSearchDispatch } from '@/lib/workspace-files/search/enqueue-dispatch' -const mockTrigger = vi.mocked(tasks.trigger) - -setEnvFlags({ isTriggerDevEnabled: true }) afterAll(resetEnvFlagsMock) describe('workspace file search dispatcher enqueue', () => { beforeEach(() => { vi.useFakeTimers() vi.setSystemTime(new Date('2026-08-29T12:34:45.000Z')) - asyncJobsRegionMockFns.mockResolveTriggerRegion.mockResolvedValue('us-east-1') - mockTrigger.mockResolvedValue({ id: 'run-1' }) + setEnvFlags({ isTriggerDevEnabled: true }) + triggerSdkMockFns.mockTasksTrigger.mockImplementation(createIdempotentTasksTrigger()) + mocks.hasWork.mockReset() + mocks.hasWork.mockResolvedValue(true) }) afterEach(() => { vi.useRealTimers() }) - it('waits only for durable Trigger.dev acceptance and does not run the dispatcher inline', async () => { + it('returns the Trigger.dev run once it is accepted', async () => { await expect(enqueueWorkspaceFileSearchDispatch()).resolves.toEqual({ + triggered: true, backend: 'trigger-dev', jobId: 'run-1', }) + }) + + it('folds ticks into the run of the window their work check was made in', async () => { + const first = await enqueueWorkspaceFileSearchDispatch() + mocks.hasWork.mockImplementationOnce(async () => { + vi.advanceTimersByTime(60_000) + return true + }) + const checkedAcrossTheBoundary = await enqueueWorkspaceFileSearchDispatch() + const nextWindow = await enqueueWorkspaceFileSearchDispatch() - expect(mockTrigger).toHaveBeenCalledWith('workspace-file-search-dispatch', undefined, { - idempotencyKey: 'workspace-file-search-dispatch:29800114', - idempotencyKeyTTL: '5m', - maxDuration: 60, - region: 'us-east-1', - ttl: '5m', + expect(first.jobId).toBe('run-1') + expect(checkedAcrossTheBoundary.jobId).toBe('run-1') + expect(nextWindow.jobId).toBe('run-2') + }) + + it.each([ + { backend: 'trigger-dev', triggerDevEnabled: true }, + { backend: 'inline', triggerDevEnabled: false }, + ])( + 'starts no $backend dispatcher run when there is no dispatch work', + async ({ triggerDevEnabled }) => { + setEnvFlags({ isTriggerDevEnabled: triggerDevEnabled }) + mocks.hasWork.mockResolvedValue(false) + + await expect(enqueueWorkspaceFileSearchDispatch()).resolves.toEqual({ + triggered: false, + backend: null, + jobId: null, + }) + } + ) + + it('still starts a dispatcher run when the work check fails', async () => { + mocks.hasWork.mockRejectedValue(new Error('statement timeout')) + + await expect(enqueueWorkspaceFileSearchDispatch()).resolves.toEqual({ + triggered: true, + backend: 'trigger-dev', + jobId: 'run-1', }) - expect(mocks.dispatch).not.toHaveBeenCalled() - expect(backgroundTaskMockFns.mockRunDetached).not.toHaveBeenCalled() }) }) diff --git a/apps/sim/lib/workspace-files/search/enqueue-dispatch.ts b/apps/sim/lib/workspace-files/search/enqueue-dispatch.ts index 750fd96d56a..794074e1fac 100644 --- a/apps/sim/lib/workspace-files/search/enqueue-dispatch.ts +++ b/apps/sim/lib/workspace-files/search/enqueue-dispatch.ts @@ -1,43 +1,50 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { type ScheduledPassResult, startScheduledPass } from '@/lib/core/async-jobs/scheduled-pass' import { isTriggerDevEnabled } from '@/lib/core/config/env-flags' import { runDetached } from '@/lib/core/utils/background' import { FILE_SEARCH_DISPATCH_INTERVAL_MS, FILE_SEARCH_DISPATCH_MAX_DURATION_SECONDS, } from '@/lib/workspace-files/search/constants' -import { dispatchWorkspaceFileSearchIndexJobs } from '@/lib/workspace-files/search/dispatcher' +import { + dispatchWorkspaceFileSearchIndexJobs, + hasWorkspaceFileSearchDispatchWork, +} from '@/lib/workspace-files/search/dispatcher' + +const logger = createLogger('WorkspaceFileSearchDispatchEnqueue') -export interface WorkspaceFileSearchDispatchEnqueueResult { - backend: 'trigger-dev' | 'inline' - jobId: string | null +/** + * Starts no dispatcher run when there is nothing to dispatch. A failed check starts one anyway, so + * an unhealthy probe can delay indexing by at most the dispatcher's own failure, never strand it. + */ +async function hasDispatchWork(now: Date): Promise { + try { + return await hasWorkspaceFileSearchDispatchWork(now) + } catch (error) { + logger.warn('Workspace file search dispatch work check failed; dispatching anyway', { + error: getErrorMessage(error), + }) + return true + } } /** * Durably hands a dispatcher run to Trigger.dev and returns after acceptance. The inline branch is * development-only and detaches from the HTTP response because the local server is long-lived. */ -export async function enqueueWorkspaceFileSearchDispatch(): Promise { - if (!isTriggerDevEnabled) { - runDetached('workspace-file-search-dispatch', dispatchWorkspaceFileSearchIndexJobs) - return { backend: 'inline', jobId: null } - } - - const [{ tasks }, { workspaceFileSearchDispatchTask }, { resolveTriggerRegion }] = - await Promise.all([ - import('@trigger.dev/sdk'), - import('@/background/workspace-file-search-dispatch'), - import('@/lib/core/async-jobs/region'), - ]) - const scheduleWindow = Math.floor(Date.now() / FILE_SEARCH_DISPATCH_INTERVAL_MS) - const handle = await tasks.trigger( - 'workspace-file-search-dispatch', - undefined, - { - idempotencyKey: `workspace-file-search-dispatch:${scheduleWindow}`, - idempotencyKeyTTL: '5m', - maxDuration: FILE_SEARCH_DISPATCH_MAX_DURATION_SECONDS, - region: await resolveTriggerRegion(), - ttl: '5m', - } - ) - return { backend: 'trigger-dev', jobId: handle.id } +export async function enqueueWorkspaceFileSearchDispatch(): Promise { + const now = new Date() + return startScheduledPass({ + due: await hasDispatchWork(now), + triggerAvailable: () => isTriggerDevEnabled, + startInline: () => + runDetached('workspace-file-search-dispatch', dispatchWorkspaceFileSearchIndexJobs), + trigger: { + taskId: 'workspace-file-search-dispatch', + intervalMs: FILE_SEARCH_DISPATCH_INTERVAL_MS, + at: now, + options: { maxDuration: FILE_SEARCH_DISPATCH_MAX_DURATION_SECONDS, ttl: '5m' }, + }, + }) } diff --git a/apps/sim/lib/workspace-files/search/index-state.ts b/apps/sim/lib/workspace-files/search/index-state.ts index 02835dbeb95..1aa4975a4d8 100644 --- a/apps/sim/lib/workspace-files/search/index-state.ts +++ b/apps/sim/lib/workspace-files/search/index-state.ts @@ -263,6 +263,12 @@ export async function failFileSearchRevision( }) } +/** + * A build {@link cleanupFileSearchBuilds} reclaims: retired, or abandoned past its lease. Implies + * `expires_at IS NOT NULL`, so `workspace_file_search_build_cleanup_idx` serves it. + */ +export const fileSearchBuildExpired = sql`${workspaceFileSearchBuild.expiresAt} <= now()` + /** Expired and invalidated builds drain without cascading a large delete through file mutations. */ export async function cleanupFileSearchBuilds(): Promise { const deadline = Date.now() + FILE_SEARCH_CLEANUP_BUDGET_MS @@ -277,7 +283,7 @@ export async function cleanupFileSearchBuilds(): Promise { const builds = await tx.execute<{ id: string }>(sql`SELECT id FROM workspace_file_search_build - WHERE expires_at <= now() ORDER BY expires_at, id LIMIT ${FILE_SEARCH_CLEANUP_BATCH_BUILDS} FOR UPDATE SKIP LOCKED`) + WHERE ${fileSearchBuildExpired} ORDER BY expires_at, id LIMIT ${FILE_SEARCH_CLEANUP_BATCH_BUILDS} FOR UPDATE SKIP LOCKED`) if (!builds.length) return null const buildIds = sql.join( builds.map((build) => sql`${build.id}`), diff --git a/packages/testing/src/mocks/outbox-service.mock.ts b/packages/testing/src/mocks/outbox-service.mock.ts index 210dfc118fb..15f80918643 100644 --- a/packages/testing/src/mocks/outbox-service.mock.ts +++ b/packages/testing/src/mocks/outbox-service.mock.ts @@ -69,6 +69,7 @@ export const outboxServiceMockFns = { }), mockFindDeadLetteredEvents: vi.fn(), mockHasInflightOutboxEvent: vi.fn(), + mockHasDueOutboxWork: vi.fn(), mockProcessOutboxEvents: vi.fn(), mockProcessOutboxEventById: vi.fn(), } @@ -97,6 +98,7 @@ export const outboxServiceMock = { outboxPayloadHasSourceOperationId: outboxServiceMockFns.mockOutboxPayloadHasSourceOperationId, findDeadLetteredEvents: outboxServiceMockFns.mockFindDeadLetteredEvents, hasInflightOutboxEvent: outboxServiceMockFns.mockHasInflightOutboxEvent, + hasDueOutboxWork: outboxServiceMockFns.mockHasDueOutboxWork, processOutboxEvents: outboxServiceMockFns.mockProcessOutboxEvents, processOutboxEventById: outboxServiceMockFns.mockProcessOutboxEventById, } diff --git a/packages/testing/src/mocks/trigger-sdk.mock.ts b/packages/testing/src/mocks/trigger-sdk.mock.ts index ce12e0c4f7c..678c2d26709 100644 --- a/packages/testing/src/mocks/trigger-sdk.mock.ts +++ b/packages/testing/src/mocks/trigger-sdk.mock.ts @@ -83,6 +83,31 @@ function createMockTask(config: T) { } } +/** + * A `tasks.trigger` implementation that deduplicates like Trigger.dev: a trigger whose task already + * holds its `idempotencyKey` returns that run's id, and a new key (or no key) starts a new run + * (`run-1`, `run-2`, …). Key TTLs are not modelled: a key is held for the fake's lifetime. + * + * @example + * ```ts + * triggerSdkMockFns.mockTasksTrigger.mockImplementation(createIdempotentTasksTrigger()) + * ``` + */ +export function createIdempotentTasksTrigger() { + const runIdByKey = new Map() + let runCount = 0 + return async (taskId: string, _payload?: unknown, options?: unknown): Promise => { + const key = (options as { idempotencyKey?: unknown } | undefined)?.idempotencyKey + const scopedKey = typeof key === 'string' ? JSON.stringify([taskId, key]) : null + const existing = scopedKey ? runIdByKey.get(scopedKey) : undefined + if (existing) return { id: existing } + runCount += 1 + const id = `run-${runCount}` + if (scopedKey) runIdByKey.set(scopedKey, id) + return { id } + } +} + /** * Controllable mock functions for `@trigger.dev/sdk` (globally mocked in `apps/sim/vitest.setup.ts`). *