diff --git a/.agents/skills/add-enrichment/SKILL.md b/.agents/skills/add-enrichment/SKILL.md index fcb83525ab9..35ef4596b24 100644 --- a/.agents/skills/add-enrichment/SKILL.md +++ b/.agents/skills/add-enrichment/SKILL.md @@ -22,7 +22,7 @@ Because enrichments run on Sim's hosted keys by default, **every provider tool y ## Architecture (what you're plugging into) -- **`enrichments/types.ts`** — `EnrichmentConfig { id, name, description, icon, inputs, outputs, providers }` and `EnrichmentProvider { id, label, toolId, buildParams, projectFailure, mapOutput }` — `toolProvider` fills `projectFailure` with the standard HTTP projection; override it only for a provider whose failure shape is nonstandard (see `enrichments/provider-failures/`). Providers are **plain data** (no `@/tools` import) so the catalog stays client-safe. +- **`enrichments/types.ts`** — `EnrichmentConfig { id, name, description, icon, inputs, outputs, providers }` and `EnrichmentProvider { id, label, toolId, buildParams, projectFailure, mapOutput, isVerified? }` — `toolProvider` fills `projectFailure` with the standard HTTP projection; override it only for a provider whose failure shape is nonstandard (see `enrichments/provider-failures/`). Providers are **plain data** (no `@/tools` import) so the catalog stays client-safe. - **`enrichments/providers.ts`** — `toolProvider(...)` (typed passthrough) plus shared input helpers: `str(v)`, `normalizeDomain(v)`, `firstNonEmpty(arr)`, `splitName(fullName)`, and `projectEnrichmentProviderFailure`. - **`enrichments/run.ts`** — the server-only cascade runner. Calls `executeTool(provider.toolId, { ...params, _context: { workspaceId, userId } })`, accumulates hosted-key cost, returns the first non-empty mapped result. **You do not edit this** — it works for any registry entry. - **`enrichments/registry.ts`** — `ENRICHMENT_REGISTRY` / `ALL_ENRICHMENTS` / `getEnrichment`. Register new entries here. @@ -37,7 +37,9 @@ For each output the enrichment produces, decide which existing tool provides it. - Its `params` accept what you can derive from table columns (read the tool's `params`). - Its `outputs` / `transformResponse` actually expose the field you need (read the real output shape — don't assume). -Order providers **cheapest / most-likely-to-hit first**; the cascade stops at the first non-empty result. Apollo and LinkedIn APIs are not hosted-safe (ToS) — never call them as providers. +Order providers by **accuracy first, then price per hit**. Order decides which answer wins when two providers would disagree, so a cheaper but less accurate provider ahead of a better one trades quality for cost; reorder for price only with benchmark evidence. Among similarly accurate providers, put the ones that bill on a miss or charge a high price per hit after the cheaper, free-on-miss ones (check each vendor's billing docs and the tool's `hosting.pricing`); never move a more accurate provider behind a less accurate one to save cost. The cascade stops at the first non-empty, verified result. Apollo and LinkedIn APIs are not hosted-safe (ToS) — never call them as providers. + +Prefer a vendor's own verified-only option (e.g. Prospeo `only_verified_email`) when it turns an unverified result into a free miss. Otherwise, when a provider can return an unconfirmed hit (a catch-all guess, a `risky` status), declare `isVerified(output)`: an unconfirmed hit is held while later providers look for a confirmed one, and fills the cell only if none does. `isVerified: false` marks a provider that never confirms its hits (a database record); it runs only while nothing is held. ## Step 2: Verify hosted-key support — chain to `/add-hosted-key` if missing @@ -136,7 +138,8 @@ export const ENRICHMENT_REGISTRY: EnrichmentRegistry = { - [ ] Each output mapped to a real tool field (verified against the tool's `params`/`outputs`) - [ ] **Every provider tool has a `hosting` block — ran `/add-hosted-key` for any that didn't** -- [ ] Providers ordered cheapest / most-likely-first; Apollo/LinkedIn not used +- [ ] Providers ordered accuracy-first, then price per hit (billing on a miss checked against vendor docs); Apollo/LinkedIn not used +- [ ] Providers that can return an unconfirmed hit declare `isVerified` - [ ] Enrichment file is client-safe (no `@/tools` import); uses `toolProvider` + shared helpers - [ ] `buildParams` returns `null` on insufficient inputs; `mapOutput` returns `null` on a miss - [ ] Registered in `enrichments/registry.ts` diff --git a/apps/sim/background/enrichment-capability-subject.test.ts b/apps/sim/background/enrichment-capability-subject.test.ts index aebf0a37144..52aa7b0d978 100644 --- a/apps/sim/background/enrichment-capability-subject.test.ts +++ b/apps/sim/background/enrichment-capability-subject.test.ts @@ -7,6 +7,10 @@ import { billingUsageGateCacheMock, billingUsageGateCacheMockFns, } from '@sim/testing/mocks/billing-usage-gate-cache.mock' +import { + billingUsageLogMock, + billingUsageLogMockFns, +} from '@sim/testing/mocks/billing-usage-log.mock' import { executeWorkflowMock } from '@sim/testing/mocks/execute-workflow.mock' import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' import { tableEventsMock } from '@sim/testing/mocks/table-events.mock' @@ -60,6 +64,7 @@ vi.mock('@/enrichments/run', () => ({ })) vi.mock('@/lib/billing/core/billing-attribution', () => billingAttributionMock) vi.mock('@/lib/billing/core/usage-gate-cache', () => billingUsageGateCacheMock) +vi.mock('@/lib/billing/core/usage-log', () => billingUsageLogMock) vi.mock('@/lib/table/rows/secret-provenance', () => tableRowsSecretProvenanceMock) vi.mock('@/executor/utils/resolved-secret-trace-registry', () => ({ ResolvedSecretTraceRegistry: class { @@ -112,7 +117,7 @@ const GROUP = { type: 'enrichment' as const, enrichmentId: 'company-lookup', workflowId: '', - outputs: [{ columnName: 'col-out', blockId: '', path: '' }], + outputs: [{ columnName: 'col-out', blockId: '', path: '', outputId: 'email' }], inputMappings: [{ columnName: 'col-in', inputName: 'domain' }], } @@ -288,4 +293,24 @@ describe('enrichment cell capability subject', () => { await runRowCascadeLoop(payload(null, 'legacy-trigger-user') as never) expect(gatedUserId()).toBeNull() }) + + /** A completed cell is final on the auto path, so an outage must not complete blank. */ + it('marks the cell errored when every provider errored', async () => { + mocks.runEnrichment.mockResolvedValue({ + result: {}, + cost: 0.02, + error: 'Rate limit exceeded', + detail: { providers: [] }, + }) + + await runRowCascadeLoop(payload('acting-user', 'acting-user') as never) + + const terminal = mocks.writeWorkflowGroupState.mock.calls.at(-1) as unknown[] + const write = terminal[1] as { executionState: Record; dataPatch?: unknown } + const state = write.executionState + expect(state.status).toBe('error') + expect(state.error).toBe('Rate limit exceeded') + expect(write.dataPatch).toEqual({ 'col-out': '' }) + expect(billingUsageLogMockFns.mockRecordUsage).toHaveBeenCalledTimes(1) + }) }) diff --git a/apps/sim/background/workflow-column-execution.ts b/apps/sim/background/workflow-column-execution.ts index 7cd569ce424..95751e02da0 100644 --- a/apps/sim/background/workflow-column-execution.ts +++ b/apps/sim/background/workflow-column-execution.ts @@ -681,27 +681,29 @@ async function runWorkflowAndWriteTerminal( await enrichmentRegistry.importCrossingProvenance(inputProvenance, enrichInputs, { trusted: true, }) - const { result, cost, detail } = await withResourceOutboundScope({ workspaceId }, () => - runEnrichment(enrichment, enrichInputs, { - tableId, - rowId, - workspaceId, - /** - * The person who asked, not who pays. `triggeredByUserId` is an - * attribution: for a workspace-API-key run it names the workspace's - * billing owner, and running that bystander's tool denylist against - * an actorless request is wrong in both directions — it fails cells - * nobody meant to govern, and it skips the denylist for the person - * who actually triggered one. The governed subject is carried - * separately from the dispatch. `null` means no per-tool gate - * applies, which is the documented behavior for an actorless run — - * stated, because the field is required precisely so it cannot be - * skipped by omission. - */ - userId: payload.capabilityGovernedUserId ?? null, - signal: attemptSignal, - resolvedSecretTraceRegistry: enrichmentRegistry, - }) + const { result, cost, error, detail } = await withResourceOutboundScope( + { workspaceId }, + () => + runEnrichment(enrichment, enrichInputs, { + tableId, + rowId, + workspaceId, + /** + * The person who asked, not who pays. `triggeredByUserId` is an + * attribution: for a workspace-API-key run it names the workspace's + * billing owner, and running that bystander's tool denylist against + * an actorless request is wrong in both directions — it fails cells + * nobody meant to govern, and it skips the denylist for the person + * who actually triggered one. The governed subject is carried + * separately from the dispatch. `null` means no per-tool gate + * applies, which is the documented behavior for an actorless run — + * stated, because the field is required precisely so it cannot be + * skipped by omission. + */ + userId: payload.capabilityGovernedUserId ?? null, + signal: attemptSignal, + resolvedSecretTraceRegistry: enrichmentRegistry, + }) ) // An abort during the cascade must not be recorded as a completed cell. @@ -720,7 +722,7 @@ async function runWorkflowAndWriteTerminal( /** * Record the triggerer or system fallback as actor while charging the - * exact workspace payer. Billing failures do not fail a successful cell. + * exact workspace payer. Billing failures do not fail the cell. */ if (cost > 0) { try { @@ -753,7 +755,8 @@ async function runWorkflowAndWriteTerminal( // Write every output column: the result value when present, else clear // it. A partial/empty result must blank the columns it didn't fill so a - // re-run that finds less than before doesn't leave stale values. + // re-run that finds less than before doesn't leave stale values, and a + // failed run must not leave old values for dependent groups to consume. const dataPatch: RowData = {} for (const out of group.outputs) { if (!out.outputId) continue @@ -761,20 +764,22 @@ async function runWorkflowAndWriteTerminal( dataPatch[out.columnName] = value === undefined || value === null ? '' : (value as RowData[string]) } + // Every provider that ran errored (rate limit, missing hosted key, auth): + // `error` rather than `completed`, so a manual re-run picks the cell up. await writeState( { - status: 'completed', + status: error ? 'error' : 'completed', executionId, jobId: null, workflowId: statusId, - error: null, + error, enrichmentDetails: detail, }, dataPatch, undefined, createTableRowSecretProvenanceFromRegistry(dataPatch, enrichmentRegistry) ) - return 'completed' + return error ? 'error' : 'completed' } catch (err) { await writeState({ status: 'error', diff --git a/apps/sim/enrichments/email-verification/email-verification.ts b/apps/sim/enrichments/email-verification/email-verification.ts index e9c8048689c..7edd4f5453f 100644 --- a/apps/sim/enrichments/email-verification/email-verification.ts +++ b/apps/sim/enrichments/email-verification/email-verification.ts @@ -2,14 +2,26 @@ import { ShieldCheck } from '@sim/emcn/icons' import { str, toolProvider } from '@/enrichments/providers' import type { EnrichmentConfig } from '@/enrichments/types' +/** + * Verdicts that say nothing about the mailbox; the cascade falls through to the + * next verifier on these. MillionVerifier reports `unverified` (and may report + * `error`) when it could not complete a check. + */ +const INCONCLUSIVE_STATUSES = new Set(['unknown', 'unverified', 'error']) + +function mapVerdict(output: Record) { + const status = str(output.status) + if (!status || INCONCLUSIVE_STATUSES.has(status)) return null + return { status, deliverable: output.deliverable === true } +} + /** * Email Verification enrichment. Checks an email address's deliverability via a * verifier waterfall — ZeroBounce first (highest coverage), then NeverBounce, - * then MillionVerifier, then Enrow. A provider that returns a - * definitive verdict (valid / invalid / catch_all / disposable / etc.) fills the - * cell; a provider that can only return `unknown` falls through to the next so - * the row gets the most confident answer available. All providers support hosted - * keys. + * then MillionVerifier, then Enrow. A provider that returns a definitive verdict + * (valid / invalid / catch_all / disposable / etc.) fills the cell; an + * inconclusive one falls through to the next so the row gets the most confident + * answer available. All providers support hosted keys. */ export const emailVerificationEnrichment: EnrichmentConfig = { id: 'email-verification', @@ -31,12 +43,7 @@ export const emailVerificationEnrichment: EnrichmentConfig = { if (!email) return null return { email } }, - mapOutput: (output) => { - const status = str(output.status) - // Fall through to the next verifier when the verdict is missing or inconclusive. - if (!status || status === 'unknown') return null - return { status, deliverable: output.deliverable === true } - }, + mapOutput: mapVerdict, }), toolProvider({ id: 'neverbounce', @@ -47,11 +54,7 @@ export const emailVerificationEnrichment: EnrichmentConfig = { if (!email) return null return { email } }, - mapOutput: (output) => { - const status = str(output.status) - if (!status || status === 'unknown') return null - return { status, deliverable: output.deliverable === true } - }, + mapOutput: mapVerdict, }), toolProvider({ id: 'millionverifier', @@ -62,11 +65,7 @@ export const emailVerificationEnrichment: EnrichmentConfig = { if (!email) return null return { email } }, - mapOutput: (output) => { - const status = str(output.status) - if (!status || status === 'unknown') return null - return { status, deliverable: output.deliverable === true } - }, + mapOutput: mapVerdict, }), toolProvider({ id: 'enrow', diff --git a/apps/sim/enrichments/phone-number/phone-number.ts b/apps/sim/enrichments/phone-number/phone-number.ts index 99354a63e3e..7d71ee5fe9c 100644 --- a/apps/sim/enrichments/phone-number/phone-number.ts +++ b/apps/sim/enrichments/phone-number/phone-number.ts @@ -7,11 +7,11 @@ import type { EnrichmentConfig } from '@/enrichments/types' /** * Phone Number enrichment. Finds a contact's phone number from a full name plus * any available identifiers (company domain, LinkedIn URL) via a waterfall: - * People Data Labs (name match) → Wiza reveal → Findymail (LinkedIn) → Prospeo - * mobile → LeadMagic (LinkedIn) → Datagma (LinkedIn). Each provider - * opportunistically uses whatever identifiers the row provides and self-skips - * when it has none usable, so adding more inputs widens coverage without - * reordering. First phone wins; all providers support hosted keys. + * People Data Labs (name + company or LinkedIn) → Wiza reveal → Findymail + * (LinkedIn) → Prospeo mobile → LeadMagic (LinkedIn) → Datagma (LinkedIn). Each + * provider opportunistically uses whatever identifiers the row provides and + * self-skips when it has none usable, so adding more inputs widens coverage + * without reordering. First phone wins; all providers support hosted keys. */ export const phoneNumberEnrichment: EnrichmentConfig = { id: 'phone-number', @@ -31,19 +31,23 @@ export const phoneNumberEnrichment: EnrichmentConfig = { toolId: 'pdl_person_enrich', buildParams: (inputs) => { const name = str(inputs.fullName) - if (!name) return null + const company = normalizeDomain(inputs.companyDomain) + const linkedin = str(inputs.linkedinUrl) + // A bare name is not a valid PDL match and scores below min_likelihood 6. + if (!name || !(company || linkedin)) return null // `required` makes PDL 404 (free) when the profile has no phone, // instead of charging a credit for a match we'd discard as a no-match. return filterUndefined({ name, - company: normalizeDomain(inputs.companyDomain) || undefined, + company: company || undefined, + profile: linkedin || undefined, min_likelihood: 6, required: 'phone_numbers OR mobile_phone', }) }, mapOutput: (output) => { const person = output.person as Record | undefined - const phone = firstNonEmpty(person?.phone_numbers) ?? str(person?.mobile_phone) + const phone = str(person?.mobile_phone) || firstNonEmpty(person?.phone_numbers) return phone ? { phone } : null }, }), @@ -102,6 +106,8 @@ export const phoneNumberEnrichment: EnrichmentConfig = { full_name: fullName || undefined, company_website: companyWebsite || undefined, enrich_mobile: true, + // An unverified mobile becomes a free NO_MATCH instead of a billed hit. + only_verified_mobile: true, }) }, projectFailure: projectProspeoEnrichmentFailure, diff --git a/apps/sim/enrichments/run.test.ts b/apps/sim/enrichments/run.test.ts index 421f926dd52..1f846e2bacb 100644 --- a/apps/sim/enrichments/run.test.ts +++ b/apps/sim/enrichments/run.test.ts @@ -17,6 +17,7 @@ function prov( build?: (inputs: Record) => Record | null projectFailure?: EnrichmentProvider['projectFailure'] map?: (output: Record) => Record | null + isVerified?: EnrichmentProvider['isVerified'] } = {} ): EnrichmentProvider { return toolProvider({ @@ -26,6 +27,7 @@ function prov( buildParams: opts.build ?? (() => ({ q: 'x' })), projectFailure: opts.projectFailure, mapOutput: opts.map ?? ((o) => (o.email ? { email: o.email } : null)), + isVerified: opts.isVerified, }) } @@ -201,3 +203,93 @@ describe('runEnrichment cascade detail', () => { expect(outcome.detail.providers.map((p) => p.status)).toEqual(['not_run', 'not_run']) }) }) + +describe('runEnrichment verified-first cascade', () => { + const checksVerified = (o: Record) => o.verified === true + + it('passes an unverified hit over for a later verified one', async () => { + mockExecuteTool.mockImplementation((toolId: string) => { + if (toolId === 'tool_a') + return { success: true, output: { email: 'guess@acme.com', cost: { total: 0.015 } } } + return { + success: true, + output: { email: 'jane@acme.com', verified: true, cost: { total: 0.02 } }, + } + }) + + const outcome = await runEnrichment( + config([ + prov('a', { isVerified: checksVerified }), + prov('b', { isVerified: checksVerified }), + ]), + {}, + ctx + ) + + expect(outcome.result).toEqual({ email: 'jane@acme.com' }) + expect(outcome.detail.matchedProvider).toBe('b') + expect(outcome.cost).toBeCloseTo(0.035) + expect(outcome.detail.providers.map((p) => p.status)).toEqual(['no_match', 'matched']) + }) + + it('fills the cell with the first unverified hit when nothing verifies', async () => { + mockExecuteTool.mockImplementation((toolId: string) => { + if (toolId === 'tool_a') return { success: true, output: { email: 'first@acme.com' } } + if (toolId === 'tool_b') return { success: true, output: { email: 'second@acme.com' } } + return { success: false, output: { status: 404 } } + }) + + const outcome = await runEnrichment( + config([ + prov('a', { isVerified: checksVerified }), + prov('b', { isVerified: checksVerified }), + prov('c'), + ]), + {}, + ctx + ) + + expect(outcome.result).toEqual({ email: 'first@acme.com' }) + expect(outcome.provider).toBe('A') + expect(outcome.error).toBeNull() + expect(outcome.detail.matchedProvider).toBe('a') + expect(outcome.detail.providers.map((p) => p.status)).toEqual([ + 'matched', + 'no_match', + 'no_match', + ]) + }) + + it('skips a never-verifying provider once an unverified hit is held', async () => { + mockExecuteTool.mockImplementation((toolId: string) => { + if (toolId === 'tool_a') return { success: true, output: { email: 'guess@acme.com' } } + throw new Error(`${toolId} should not run while a hit is held`) + }) + + const outcome = await runEnrichment( + config([prov('a', { isVerified: checksVerified }), prov('pdl', { isVerified: false })]), + {}, + ctx + ) + + expect(outcome.result).toEqual({ email: 'guess@acme.com' }) + expect(outcome.detail.providers.map((p) => p.status)).toEqual(['matched', 'skipped']) + expect(mockExecuteTool).toHaveBeenCalledTimes(1) + }) + + it('accepts a never-verifying provider hit when nothing was held before it', async () => { + mockExecuteTool.mockImplementation((toolId: string) => { + if (toolId === 'tool_a') return { success: false, output: { status: 404 } } + return { success: true, output: { email: 'record@acme.com' } } + }) + + const outcome = await runEnrichment( + config([prov('a', { isVerified: checksVerified }), prov('pdl', { isVerified: false })]), + {}, + ctx + ) + + expect(outcome.result).toEqual({ email: 'record@acme.com' }) + expect(outcome.detail.matchedProvider).toBe('pdl') + }) +}) diff --git a/apps/sim/enrichments/run.ts b/apps/sim/enrichments/run.ts index 93a1e6e7eac..7761b108742 100644 --- a/apps/sim/enrichments/run.ts +++ b/apps/sim/enrichments/run.ts @@ -67,12 +67,14 @@ function readCost(output: Record): number { /** * Runs an enrichment's provider cascade for one row. Tries providers in order; - * the first that returns a non-empty result wins and is returned. A provider is - * skipped when its `buildParams` returns `null` (insufficient inputs); a tool - * failure or empty mapped result falls through to the next. When every provider - * that ran errored, `error` is set so the caller can mark the cell errored; a - * clean miss leaves `error: null` (blank cell). Hosted-key cost is accumulated - * across providers for the caller to bill. + * the first that returns a non-empty result wins and is returned. A hit its + * provider marks unverified (`isVerified` → false) is held while the cascade + * looks for a verified one, and fills the cell only if none turns up. A + * provider is skipped when its `buildParams` returns `null` (insufficient + * inputs); a tool failure or empty mapped result falls through to the next. + * When every provider that ran errored, `error` is set so the caller can mark + * the cell errored; a clean miss leaves `error: null` (blank cell). Hosted-key + * cost is accumulated across providers for the caller to bill. * * Server-only: imports `executeTool`, which pulls in DB / mailer code. Only the * background cell executor imports this module (dynamically). @@ -88,13 +90,18 @@ export async function runEnrichment( let lastError: string | null = null let matchedProvider: string | null = null let winner: { result: Record; label: string } | null = null + let fallback: { + result: Record + label: string + outcome: EnrichmentProviderOutcome + } | null = null const providers: EnrichmentProviderOutcome[] = [] const startedAt = Date.now() for (let i = 0; i < enrichment.providers.length; i++) { const provider = enrichment.providers[i] if (ctx.signal?.aborted) break - const params = provider.buildParams(inputs) + const params = fallback && provider.isVerified === false ? null : provider.buildParams(inputs) if (!params) { providers.push({ id: provider.id, @@ -140,7 +147,12 @@ export async function runEnrichment( const providerCost = readCost(response.output) cost += providerCost const result = provider.mapOutput(response.output) - if (result && hasResult(result)) { + const hit = result !== null && hasResult(result) + const verified = + typeof provider.isVerified === 'function' + ? provider.isVerified(response.output) + : provider.isVerified !== false + if (hit && verified) { providers.push({ id: provider.id, label: provider.label, @@ -155,8 +167,9 @@ export async function runEnrichment( logger.info('Enrichment hit', { enrichmentId: enrichment.id, provider: provider.id }) break } - // Ran cleanly but mapped to nothing — a no-match, fall through to the next. - providers.push({ + // Ran cleanly but mapped to nothing, or an unconfirmed hit (the first one + // is held as a fallback): a no-match, fall through to the next. + const outcome: EnrichmentProviderOutcome = { id: provider.id, label: provider.label, toolId: provider.toolId, @@ -164,7 +177,9 @@ export async function runEnrichment( cost: providerCost, durationMs: Date.now() - providerStart, error: null, - }) + } + providers.push(outcome) + if (hit) fallback ??= { result, label: provider.label, outcome } } catch (err) { errorCount++ lastError = getErrorMessage(err) @@ -185,6 +200,16 @@ export async function runEnrichment( } } + if (!winner && fallback) { + winner = { result: fallback.result, label: fallback.label } + matchedProvider = fallback.outcome.id + fallback.outcome.status = 'matched' + logger.info('Enrichment unverified hit', { + enrichmentId: enrichment.id, + provider: matchedProvider, + }) + } + // Any provider not represented yet never ran — the cascade short-circuited on // a match or aborted mid-run. Record them as `not_run` (in registry order) so // the panel always shows the full configured cascade. diff --git a/apps/sim/enrichments/types.ts b/apps/sim/enrichments/types.ts index eba5cba3fc0..6d61b23ad72 100644 --- a/apps/sim/enrichments/types.ts +++ b/apps/sim/enrichments/types.ts @@ -63,7 +63,7 @@ export type EnrichmentProviderFailureProjection = * (which the table UI imports for metadata) never pulls in server-only tool * code. Providers are attempted in declared order (a fallback cascade); the * cascade runner (`run.ts`, server-only) calls the tool and the first provider - * to return a non-empty result fills the cell. + * to return a non-empty, verified result fills the cell. */ export interface EnrichmentProvider { /** Stable id for logs, e.g. `'hunter'`, `'pdl'`. */ @@ -87,6 +87,16 @@ export interface EnrichmentProvider { * An empty/`null` result falls through to the next provider. */ mapOutput: (output: Record) => Record | null + /** + * Whether a hit is confirmed deliverable/accurate by the provider (e.g. a + * mailbox-verified email). When it returns `false`, the hit is held as a + * fallback and the cascade keeps looking for a confirmed one; the first held + * hit fills the cell only if no later provider confirms. Omit for providers + * whose every hit is already confirmed. `false` marks a provider that never + * confirms its hits: it runs only while nothing is held, since another + * unconfirmed hit could not replace the held one. + */ + isVerified?: ((output: Record) => boolean) | false } /** diff --git a/apps/sim/enrichments/work-email/work-email.ts b/apps/sim/enrichments/work-email/work-email.ts index 37727b63941..7eb271656c9 100644 --- a/apps/sim/enrichments/work-email/work-email.ts +++ b/apps/sim/enrichments/work-email/work-email.ts @@ -1,18 +1,27 @@ import { Mail } from '@sim/emcn/icons' -import { filterUndefined } from '@sim/utils/object' +import { filterUndefined, toArray } from '@sim/utils/object' import { projectProspeoEnrichmentFailure } from '@/enrichments/provider-failures/prospeo' import { normalizeDomain, splitName, str, toolProvider } from '@/enrichments/providers' import type { EnrichmentConfig } from '@/enrichments/types' +/** The address Wiza marked `valid`: the primary one, else the first in `emails`. */ +function wizaValidEmail(output: Record): unknown { + if (output.email_status === 'valid') return output.email + const emails = toArray>(output.emails) + return emails.find((e) => e.email_status === 'valid')?.email +} + /** * Work Email enrichment. Finds a person's work email from a full name plus any * available identifiers (company domain, LinkedIn URL) via a provider waterfall: * deterministic finders first (Hunter, Findymail by name then by LinkedIn), then - * enrichment/reveal providers (Prospeo, Wiza), then People Data Labs as a broad - * record-match fallback, then Datagma, LeadMagic, Dropcontact, and Enrow - * as additional finders. Each provider opportunistically uses whatever - * identifiers the row provides and self-skips when it has none usable, so adding - * more inputs widens coverage. First email wins; all providers support hosted keys. + * enrichment/reveal providers (Prospeo, Wiza), then Datagma, LeadMagic, + * Dropcontact, and Enrow as additional finders, and People Data Labs last — an + * unverified database record at the highest price per hit. Each provider + * opportunistically uses whatever identifiers the row provides and self-skips + * when it has none usable, so adding more inputs widens coverage. The first + * mailbox-verified email wins; an unverified one (e.g. a catch-all guess) is kept + * only if nothing verified turns up. All providers support hosted keys. */ export const workEmailEnrichment: EnrichmentConfig = { id: 'work-email', @@ -40,6 +49,9 @@ export const workEmailEnrichment: EnrichmentConfig = { const email = str(output.email) return email ? { email } : null }, + // Only a mailbox-verified hit is final; accept_all/unknown guesses are held. + isVerified: (output) => + (output.verification as Record | null)?.status === 'valid', }), toolProvider({ id: 'findymail', @@ -85,6 +97,8 @@ export const workEmailEnrichment: EnrichmentConfig = { linkedin_url: linkedin || undefined, full_name: fullName || undefined, company_website: companyWebsite || undefined, + // An unverified email becomes a free NO_MATCH instead of a billed hit. + only_verified_email: true, }) }, projectFailure: projectProspeoEnrichmentFailure, @@ -110,34 +124,17 @@ export const workEmailEnrichment: EnrichmentConfig = { full_name: fullName || undefined, domain: domain || undefined, enrichment_level: 'partial', + accept_work: true, + accept_personal: false, }) }, + // Wiza bills only when a `valid` email comes back, so take that one; a + // `risky`-only result is free and held as a fallback. mapOutput: (output) => { - const email = str(output.email) - return email ? { email } : null - }, - }), - toolProvider({ - id: 'pdl', - label: 'People Data Labs', - toolId: 'pdl_person_enrich', - buildParams: (inputs) => { - const name = str(inputs.fullName) - if (!name) return null - // `required` makes PDL 404 (free) when the profile has no work email, - // instead of charging a credit for a match we'd discard as a no-match. - return filterUndefined({ - name, - company: normalizeDomain(inputs.companyDomain) || undefined, - min_likelihood: 6, - required: 'work_email', - }) - }, - mapOutput: (output) => { - const person = output.person as Record | undefined - const email = str(person?.work_email) + const email = str(wizaValidEmail(output)) || str(output.email) return email ? { email } : null }, + isVerified: (output) => str(wizaValidEmail(output)) !== '', }), toolProvider({ id: 'datagma', @@ -170,6 +167,7 @@ export const workEmailEnrichment: EnrichmentConfig = { const email = str(output.email) return email ? { email } : null }, + isVerified: (output) => output.status === 'valid', }), toolProvider({ id: 'dropcontact', @@ -190,6 +188,9 @@ export const workEmailEnrichment: EnrichmentConfig = { const email = str(output.email) return email ? { email } : null }, + // The live API labels a verified work address `nominatif@pro`; its docs say `nominative@pro`. + isVerified: (output) => + output.qualification === 'nominatif@pro' || output.qualification === 'nominative@pro', }), toolProvider({ id: 'enrow', @@ -210,5 +211,29 @@ export const workEmailEnrichment: EnrichmentConfig = { return email && qualification === 'valid' ? { email } : null }, }), + toolProvider({ + id: 'pdl', + label: 'People Data Labs', + toolId: 'pdl_person_enrich', + buildParams: (inputs) => { + const name = str(inputs.fullName) + if (!name) return null + // `required` makes PDL 404 (free) when the profile has no work email, + // instead of charging a credit for a match we'd discard as a no-match. + return filterUndefined({ + name, + company: normalizeDomain(inputs.companyDomain) || undefined, + min_likelihood: 6, + required: 'work_email', + }) + }, + mapOutput: (output) => { + const person = output.person as Record | undefined + const email = str(person?.work_email) + return email ? { email } : null + }, + // Database record, never mailbox-checked; runs only while nothing is held. + isVerified: false, + }), ], } diff --git a/apps/sim/tools/millionverifier/verify_email.ts b/apps/sim/tools/millionverifier/verify_email.ts index a8ea5a936c6..2b7f1967518 100644 --- a/apps/sim/tools/millionverifier/verify_email.ts +++ b/apps/sim/tools/millionverifier/verify_email.ts @@ -15,6 +15,13 @@ const STATUS_MAP: Record = { unverified: 'unverified', } +/** + * MillionVerifier charges only for ok, invalid, and disposable results; risky + * results (catch-all, unknown) are free — + * https://help.millionverifier.com/email-verification/email-verification-results. + */ +const BILLED_STATUSES = new Set(['valid', 'invalid', 'disposable']) + export const verifyEmailTool: ToolConfig< MillionVerifierVerifyEmailParams, MillionVerifierVerifyEmailResponse @@ -24,10 +31,9 @@ export const verifyEmailTool: ToolConfig< description: 'Verify the deliverability of an email address. Uses one verification credit.', version: '1.0.0', - hosting: millionverifierHosting(() => { - // Each verification consumes one MillionVerifier credit. - return 1 - }), + hosting: millionverifierHosting((_params, output) => + BILLED_STATUSES.has(String(output.status)) ? 1 : 0 + ), params: { email: { diff --git a/apps/sim/tools/zerobounce/verify_email.ts b/apps/sim/tools/zerobounce/verify_email.ts index f2027c697e4..e01ede4a8ee 100644 --- a/apps/sim/tools/zerobounce/verify_email.ts +++ b/apps/sim/tools/zerobounce/verify_email.ts @@ -14,7 +14,10 @@ export const verifyEmailTool: ToolConfig< description: 'Validate an email address deliverability in real time. Uses one validation credit.', version: '1.0.0', - hosting: zerobounceHosting(() => 1), + // ZeroBounce never consumes a credit for an `unknown` result. + hosting: zerobounceHosting((_params, output) => + output.status === 'unknown' ? 0 : 1 + ), params: { email: {