From 416225de7805a9361b718ec7d95d2c8159e21195 Mon Sep 17 00:00:00 2001 From: Ilya Bogin Date: Thu, 29 Oct 2026 11:07:23 +0200 Subject: [PATCH 1/2] add keenable as a built-in integration through its mcp server Keenable is signed in to through its MCP server (api.keenable.ai/mcp-oauth, OAuth with PKCE and a client registered on the spot) or connected with an API key, sent as the bearer token, as Exa and Parallel are. Agents get Search the web and Read web pages, the results written out as Exa writes its own. Custom connections to api.keenable.ai move over as the other built-in servers' do, and one reached with nothing stays custom. --- .../api/src/connections/built-in-mcp.test.ts | 1 + packages/api/src/connections/built-in-mcp.ts | 3 +- .../connections/keenable/keenable-mcp.test.ts | 43 ++++ .../src/connections/keenable/keenable-mcp.ts | 65 +++++ .../src/connections/keenable/route.test.ts | 235 ++++++++++++++++++ .../api/src/connections/keenable/route.ts | 21 ++ .../src/connections/take-over-custom.test.ts | 11 + packages/api/src/mcp/route.test.ts | 4 +- packages/api/src/records/connectors.ts | 3 +- .../api/src/records/web-connector.test.ts | 33 ++- packages/api/src/records/web-connector.ts | 18 +- .../docs/content/connections/custom-mcp.mdx | 25 +- .../docs/content/connections/keenable.mdx | 41 +++ packages/docs/content/index.mdx | 3 +- packages/docs/src/nav.ts | 1 + packages/integrations/src/catalog.ts | 10 +- packages/integrations/src/tools.test.ts | 3 +- packages/integrations/src/types.ts | 7 +- packages/web/public/integrations/keenable.svg | 8 + packages/web/src/client/connections-client.ts | 8 +- .../components/connect-with-key-dialog.tsx | 8 +- .../src/components/integration-catalog.tsx | 6 +- 22 files changed, 517 insertions(+), 40 deletions(-) create mode 100644 packages/api/src/connections/keenable/keenable-mcp.test.ts create mode 100644 packages/api/src/connections/keenable/keenable-mcp.ts create mode 100644 packages/api/src/connections/keenable/route.test.ts create mode 100644 packages/api/src/connections/keenable/route.ts create mode 100644 packages/docs/content/connections/keenable.mdx create mode 100644 packages/web/public/integrations/keenable.svg diff --git a/packages/api/src/connections/built-in-mcp.test.ts b/packages/api/src/connections/built-in-mcp.test.ts index ca343956..d47b97b1 100644 --- a/packages/api/src/connections/built-in-mcp.test.ts +++ b/packages/api/src/connections/built-in-mcp.test.ts @@ -71,6 +71,7 @@ describe('BUILT_IN_MCP', () => { test('finds the integration by host, as an address writes it', () => { expect(builtInForHost('mcp.linear.app')?.integrationId).toBe('linear'); expect(builtInForHost('Search.Parallel.ai')?.integrationId).toBe('parallel'); + expect(builtInForHost('api.keenable.ai')?.integrationId).toBe('keenable'); expect(builtInForHost('mcp.example.com')).toBeUndefined(); }); }); diff --git a/packages/api/src/connections/built-in-mcp.ts b/packages/api/src/connections/built-in-mcp.ts index a00631d8..2c0265c9 100644 --- a/packages/api/src/connections/built-in-mcp.ts +++ b/packages/api/src/connections/built-in-mcp.ts @@ -2,6 +2,7 @@ import {builtInHosts, type BuiltInMcp} from './mcp/built-in'; import {CLOUDFLARE_MCP} from './cloudflare/route'; import {EXA_MCP} from './exa/route'; import {GRANOLA_MCP} from './granola/route'; +import {KEENABLE_MCP} from './keenable/route'; import {LINEAR_MCP} from './linear/route'; import {NOTION_MCP} from './notion/route'; import {PARALLEL_MCP} from './parallel/route'; @@ -12,7 +13,7 @@ import {PARALLEL_MCP} from './parallel/route'; * its server's address, and moves custom connections to its server over when the api starts. Its * catalog entry says it is signed in to (`signIn`), for the web app. */ -export const BUILT_IN_MCP: BuiltInMcp[] = [GRANOLA_MCP, NOTION_MCP, LINEAR_MCP, CLOUDFLARE_MCP, EXA_MCP, PARALLEL_MCP]; +export const BUILT_IN_MCP: BuiltInMcp[] = [GRANOLA_MCP, NOTION_MCP, LINEAR_MCP, CLOUDFLARE_MCP, EXA_MCP, PARALLEL_MCP, KEENABLE_MCP]; /** The integration that takes over a custom MCP server at this host, if one does. */ export function builtInForHost(host: string): BuiltInMcp | undefined { diff --git a/packages/api/src/connections/keenable/keenable-mcp.test.ts b/packages/api/src/connections/keenable/keenable-mcp.test.ts new file mode 100644 index 00000000..60c61c15 --- /dev/null +++ b/packages/api/src/connections/keenable/keenable-mcp.test.ts @@ -0,0 +1,43 @@ +import {describe, expect, test} from 'bun:test'; + +import {formatKeenableResults, parseKeenablePage} from './keenable-mcp'; + +describe('formatKeenableResults', () => { + test('writes the results out as Exa writes its own, a block a page, its snippet as the highlights', () => { + const text = JSON.stringify({ + query: 'bun module mocks', + results: [ + { + url: 'https://bun.sh/docs/test/mocks', + title: 'Mocks | Bun Docs', + description: '', + snippet: 'Mocking replaces a dependency.\nUse mock.module.\n', + published_at: '2026-10-12T08:00:00Z', + acquired_at: '2026-10-13T09:00:00Z', + }, + {url: 'https://example.com', title: '', description: '', snippet: 'Example Domain', acquired_at: '2026-10-13T09:00:00Z'}, + ], + }); + expect(formatKeenableResults(text).unwrap()).toBe( + [ + 'Title: Mocks | Bun Docs\nURL: https://bun.sh/docs/test/mocks\nPublished: 2026-10-12\nHighlights:\nMocking replaces a dependency.\nUse mock.module.', + 'Title: https://example.com\nURL: https://example.com\nPublished: N/A\nHighlights:\nExample Domain', + ].join('\n\n---\n\n'), + ); + }); + + test('is Keenable out of reach when it answers with something else', () => { + expect(formatKeenableResults('Rate limit exceeded').unwrapErr().kind).toBe('provider_unreachable'); + }); +}); + +describe('parseKeenablePage', () => { + test('reads the title and the text', () => { + const text = JSON.stringify({url: 'https://example.com/', title: 'Example Domain', content: '# Example Domain\n\nThis domain is for examples.\n', description: ''}); + expect(parseKeenablePage(text).unwrap()).toEqual({title: 'Example Domain', content: '# Example Domain\n\nThis domain is for examples.'}); + }); + + test('a page without a title keeps all of it as the text', () => { + expect(parseKeenablePage(JSON.stringify({url: 'https://example.com/', content: 'Just text\n'})).unwrap()).toEqual({title: '', content: 'Just text'}); + }); +}); diff --git a/packages/api/src/connections/keenable/keenable-mcp.ts b/packages/api/src/connections/keenable/keenable-mcp.ts new file mode 100644 index 00000000..56409139 --- /dev/null +++ b/packages/api/src/connections/keenable/keenable-mcp.ts @@ -0,0 +1,65 @@ +import {type Result} from 'ts-results-es'; +import {z} from 'zod'; + +import {ApiErr, type ApiError} from '@proxy/utils'; + +import {parseToolJson} from '../mcp/mcp-client'; +import type {McpServer} from '../mcp/mcp-oauth'; + +/** + * Keenable's MCP server, at the address that takes a sign-in or an API key (its keyless one is + * `/mcp` beside it), and what its tools answer when asked for JSON: a search's results each a + * page's address, title, publish date where known and a snippet of its text, and a page read its + * title and text as Markdown. A page it couldn't read fails the tool, saying why in words: "Error + * fetching https://…: The page does not exist at this URL.". Results are written out as Exa writes + * its own, so an agent reads them all the same. Its sign-in is Keenable's Supabase authorization + * server, as the server's protected resource metadata names it. + */ + +const SERVER = 'https://api.keenable.ai/mcp-oauth'; + +const AUTH_BASE = 'https://caqmfcgrdovyjdhfnwzw.supabase.co/auth/v1'; + +export const KEENABLE: McpServer = { + name: 'Keenable', + // Keenable labels each request with the app it came from, `keenable_title` where a client can't + // set its `X-Keenable-Title` header; the tokens are for the address without it. + url: `${SERVER}?keenable_title=personal-agent-proxy`, + resource: SERVER, + registerUrl: `${AUTH_BASE}/oauth/clients/register`, + authorizeUrl: `${AUTH_BASE}/oauth/authorize`, + tokenUrl: `${AUTH_BASE}/oauth/token`, + // Supabase's own default; Keenable asks for nothing more. + scope: 'email', + toolError: (text) => ApiErr.validationError(`Keenable turned it down: ${text.trim() || 'it didn’t say why'}`), +}; + +const resultSchema = z.object({ + url: z.string(), + title: z.string().nullish(), + // `2026-10-27T00:39:34Z`, where Keenable knows it. + published_at: z.string().nullish(), + // The page's text where it fits the query, as it is, with its line breaks. Its `description` is + // always empty. + snippet: z.string().nullish(), +}); + +const searchSchema = z.object({results: z.array(resultSchema)}); + +const pageSchema = z.object({url: z.string(), title: z.string().nullish(), content: z.string().nullish()}); + +/** A `search_web_pages` answer written out as Exa writes its results: a block a page, between "---" lines. */ +export function formatKeenableResults(text: string): Result { + return parseToolJson(text, searchSchema).map(({results}) => + results + .map((result) => + [`Title: ${result.title || result.url}`, `URL: ${result.url}`, `Published: ${result.published_at?.slice(0, 10) || 'N/A'}`, 'Highlights:', (result.snippet ?? '').trim()].join('\n'), + ) + .join('\n\n---\n\n'), + ); +} + +/** A page's title and text out of what `fetch_page_content` answers. */ +export function parseKeenablePage(text: string): Result<{title: string; content: string}, ApiError> { + return parseToolJson(text, pageSchema).map((page) => ({title: page.title ?? '', content: (page.content ?? '').trim()})); +} diff --git a/packages/api/src/connections/keenable/route.test.ts b/packages/api/src/connections/keenable/route.test.ts new file mode 100644 index 00000000..8576cafd --- /dev/null +++ b/packages/api/src/connections/keenable/route.test.ts @@ -0,0 +1,235 @@ +import {beforeEach, describe, expect, mock, spyOn, test} from 'bun:test'; +import {Ok, type Result} from 'ts-results-es'; + +import type {ApiError} from '@proxy/utils'; + +const findAccountConnection = mock(); +const createConnection = mock(); +const renewConnection = mock(); +const updateConnectionName = mock(); + +const getConnectionWithCredential = mock(); + +mock.module('@proxy/db/connection', () => ({ + getConnectionWithCredential, + findAccountConnection, + createConnection, + renewConnection, + shareConnection: mock(), + updateConnectionCredential: mock(), + getConnectionCredential: mock(async () => Ok(null)), + updateConnectionTools: mock(), + updateConnectionName, +})); + +mock.module('@proxy/db/organization', () => ({getMembership: mock(async () => Ok({role: 'owner'})), listMemberNames: mock(async () => Ok([]))})); + +const requireViewer = mock(async () => Ok({orgId: 'org-1', userId: 'user-1', slug: 'org-1', name: 'Workspace', role: 'owner', onboarded: true, experiments: []})); + +mock.module('../../utils/user-org', () => ({requireViewer, requireOwner: requireViewer})); + +mock.module('../../observability/log', () => ({ + log: {debug: mock(), info: mock(), warn: mock(), error: mock()}, + serializeError: mock(() => ({})), +})); + +mock.module('../../utils/env', () => ({env: {NODE_ENV: 'production', APP_URL: 'https://app.example.com'}})); + +// The servers are on the internet. +mock.module('node:dns/promises', () => ({default: {lookup: async () => [{address: '93.184.216.34', family: 4}]}})); + +// Readable stand-ins for encryption, so a test can see what was stored. +mock.module('../../utils/secret-crypto', () => ({ + encryptSecret: (plaintext: string): Result => Ok(`enc:${plaintext}`), + decryptSecret: (stored: string): Result => Ok(stored.replace(/^enc:/, '')), +})); + +const CALLBACK = 'https://app.example.com/api/connections/keenable/callback'; + +const AUTH_BASE = 'https://caqmfcgrdovyjdhfnwzw.supabase.co/auth/v1'; + +const SERVER = 'https://api.keenable.ai/mcp-oauth?keenable_title=personal-agent-proxy'; + +// Keenable's server and its authorization server: each request, and its answers. +const keenable = {requests: [] as Array<{url: URL; body: string}>}; + +spyOn(globalThis, 'fetch').mockImplementation((async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url); + keenable.requests.push({url, body: typeof init?.body === 'string' ? init.body : ''}); + if (url.href === `${AUTH_BASE}/oauth/clients/register`) { + return Response.json({client_id: 'client-1'}, {status: 201}); + } + if (url.href === SERVER) { + // Keenable's server turns a bad key down at the door. + if (new Headers(init?.headers).get('authorization') === 'Bearer bad-key') { + return new Response('Unauthorized', {status: 401}); + } + return Response.json({jsonrpc: '2.0', id: 1, result: {tools: [{name: 'search_web_pages', inputSchema: {type: 'object'}}]}}); + } + return Response.json({access_token: 'key-1', token_type: 'bearer'}); +}) as typeof fetch); + +// The sign-in routes the api makes of Keenable's declaration. +async function routes() { + const {mcpConnectRoutes} = await import('../mcp/mcp-connect'); + const {KEENABLE_MCP} = await import('./route'); + return mcpConnectRoutes(KEENABLE_MCP); +} + +function makeRes() { + const cookies: Array<{name: string; value: string; options: Record}> = []; + const res = { + cookies, + redirect: mock(), + cookie: mock((name: string, value: string, options: Record) => { + cookies.push({name, value, options}); + }), + clearCookie: mock(), + status: mock(() => res), + json: mock(() => res), + }; + return res; +} + +const user = {userId: 'user-1', email: 'alex@example.com'}; + +async function start(query: Record = {}) { + const {handleStart: handleStartKeenableRoute} = await routes(); + const res = makeRes(); + await handleStartKeenableRoute({user, params: {org: 'org-1'}, query, headers: {}} as never, res as never); + const cookie = res.cookies.find((candidate) => candidate.name === 'proxy.keenable-state'); + const authorizeUrl = new URL(res.redirect.mock.calls[0]?.[0] as string); + return {cookie, authorizeUrl, state: authorizeUrl.searchParams.get('state') ?? ''}; +} + +async function callback(args: {cookie: string; query: Record}) { + const {handleCallback: handleKeenableCallbackRoute} = await routes(); + const res = makeRes(); + await handleKeenableCallbackRoute({user, params: {org: 'org-1'}, query: args.query, headers: {cookie: `proxy.keenable-state=${args.cookie}`}} as never, res as never); + return res; +} + +// What a connection made with a key answers, from what was saved. +async function connectWithKey(body: unknown) { + const {keyConnectRoute} = await import('../mcp/mcp-connect'); + const {KEENABLE_MCP} = await import('./route'); + return keyConnectRoute(KEENABLE_MCP)({user, body} as never); +} + +const existing = { + id: 'conn-0', + integrationId: 'keenable', + orgId: 'org-1', + ownerUserId: 'user-1', + connectedById: 'user-1', + account: 'Signed in', + createdAt: new Date('2026-10-01T00:00:00Z'), + tools: null, + name: 'Team Keenable', + defaults: [{actionId: 'search'}], + credential: 'enc:{}', +}; + +beforeEach(() => { + mock.clearAllMocks(); + keenable.requests = []; + findAccountConnection.mockResolvedValue(Ok(null)); + createConnection.mockImplementation(async (data: {integrationId: string; account: string; defaults: string[]; name?: string}) => + Ok({id: 'conn-1', integrationId: data.integrationId, account: data.account, createdAt: new Date(), tools: null, name: data.name ?? null, defaults: data.defaults.map((actionId) => ({actionId}))}), + ); + getConnectionWithCredential.mockResolvedValue(Ok(null)); + renewConnection.mockResolvedValue(Ok(undefined)); +}); + +describe('handleStartKeenableRoute', () => { + test("registers with Keenable's authorization server and sends the browser to its sign-in with PKCE, for the server's address without its query", async () => { + const {cookie, authorizeUrl} = await start(); + + expect(keenable.requests[0]?.url.href).toBe(`${AUTH_BASE}/oauth/clients/register`); + expect(JSON.parse(keenable.requests[0]?.body ?? '{}')).toMatchObject({redirect_uris: [CALLBACK], token_endpoint_auth_method: 'none'}); + expect(authorizeUrl.origin + authorizeUrl.pathname).toBe(`${AUTH_BASE}/oauth/authorize`); + expect(authorizeUrl.searchParams.get('client_id')).toBe('client-1'); + expect(authorizeUrl.searchParams.get('scope')).toBe('email'); + expect(authorizeUrl.searchParams.get('code_challenge_method')).toBe('S256'); + expect(authorizeUrl.searchParams.get('resource')).toBe('https://api.keenable.ai/mcp-oauth'); + expect(cookie?.options).toMatchObject({httpOnly: true, path: '/api/connections/keenable'}); + }); +}); + +describe('handleKeenableCallbackRoute', () => { + test('makes the connection, agents searching the web and reading pages', async () => { + const {cookie, state} = await start(); + const res = await callback({cookie: cookie?.value ?? '', query: {state, code: 'code-1'}}); + + expect(keenable.requests.at(-1)?.url.href).toBe(`${AUTH_BASE}/oauth/token`); + const created = createConnection.mock.calls[0]?.[0]; + expect(created).toMatchObject({orgId: 'org-1', integrationId: 'keenable', account: 'Signed in', defaults: ['search', 'fetch']}); + expect(JSON.parse(String(created.credential).replace(/^enc:/, ''))).toMatchObject({clientId: 'client-1', refreshToken: null, expiresAt: null}); + expect(res.redirect).toHaveBeenCalledWith('https://app.example.com/org-1/connections/conn-1?added'); + }); + + test('signing in from the catalog again makes another connection, since Keenable says nothing of who signed in', async () => { + const {cookie, state} = await start(); + await callback({cookie: cookie?.value ?? '', query: {state, code: 'code-1'}}); + + expect(findAccountConnection).not.toHaveBeenCalled(); + expect(createConnection).toHaveBeenCalledTimes(1); + }); + + test("begun from a connection's page (`?connection=`), the sign-in gives that connection the new tokens, its name kept", async () => { + getConnectionWithCredential.mockResolvedValue(Ok(existing)); + const {cookie, state} = await start({connection: 'conn-0'}); + const res = await callback({cookie: cookie?.value ?? '', query: {state, code: 'code-1'}}); + + expect(getConnectionWithCredential).toHaveBeenCalledWith(expect.objectContaining({orgId: 'org-1', userId: 'user-1'}), 'conn-0'); + expect(createConnection).not.toHaveBeenCalled(); + expect(renewConnection.mock.calls[0]?.[0]).toMatchObject({connectionId: 'conn-0', account: 'Signed in'}); + expect(updateConnectionName).not.toHaveBeenCalled(); + expect(res.redirect).toHaveBeenCalledWith('https://app.example.com/org-1/connections/conn-0'); + }); + + test('a sign-in turned down at Keenable goes back to the catalog', async () => { + const {cookie, state} = await start(); + const res = await callback({cookie: cookie?.value ?? '', query: {state, error: 'access_denied'}}); + + expect(res.redirect).toHaveBeenCalledWith('https://app.example.com/org-1/connections/new?error=keenable'); + expect(createConnection).not.toHaveBeenCalled(); + }); +}); + +describe('keyConnectRoute', () => { + test('connects with an API key, checked by listing the tools with it: the key its credential, its end its account', async () => { + const connection = (await connectWithKey({key: 'keen_a1_1234'})).unwrap(); + + expect(keenable.requests.at(-1)?.url.href).toBe(SERVER); + expect(JSON.parse(keenable.requests.at(-1)?.body ?? '{}')).toMatchObject({method: 'tools/list'}); + const created = createConnection.mock.calls[0]?.[0]; + expect(created).toMatchObject({orgId: 'org-1', integrationId: 'keenable', account: 'API key ending 1234', defaults: ['search', 'fetch']}); + expect(created.name).toBeUndefined(); + expect(JSON.parse(String(created.credential).replace(/^enc:/, ''))).toEqual({auth: 'key', key: 'keen_a1_1234'}); + expect(connection).toMatchObject({id: 'conn-1', name: null, unavailable: []}); + }); + + test('a key Keenable turns down is refused, and nothing saved', async () => { + const error = (await connectWithKey({key: 'bad-key'})).unwrapErr(); + + expect(error.kind).toBe('credentials_rejected'); + expect(createConnection).not.toHaveBeenCalled(); + }); + + test("given a connection (`connection`), the key replaces that connection's credential, its name kept", async () => { + getConnectionWithCredential.mockResolvedValue(Ok(existing)); + const connection = (await connectWithKey({connection: 'conn-0', key: 'keen_a2_5678'})).unwrap(); + + expect(createConnection).not.toHaveBeenCalled(); + expect(renewConnection.mock.calls[0]?.[0]).toMatchObject({connectionId: 'conn-0', account: 'API key ending 5678', credential: `enc:${JSON.stringify({auth: 'key', key: 'keen_a2_5678'})}`}); + expect(updateConnectionName).not.toHaveBeenCalled(); + expect(connection).toMatchObject({id: 'conn-0', name: 'Team Keenable'}); + }); + + test('a key alone makes a connection with no name of its own, and a key has no spaces', async () => { + const connection = (await connectWithKey({key: 'keen_a3_9999'})).unwrap(); + expect(connection).toMatchObject({name: null, account: 'API key ending 9999'}); + expect((await connectWithKey({key: 'Basic abc'})).unwrapErr().kind).toBe('validation_error'); + }); +}); diff --git a/packages/api/src/connections/keenable/route.ts b/packages/api/src/connections/keenable/route.ts new file mode 100644 index 00000000..79896b1a --- /dev/null +++ b/packages/api/src/connections/keenable/route.ts @@ -0,0 +1,21 @@ +import {textParam, type BuiltInMcp} from '../mcp/built-in'; +import {listMcpTools} from '../mcp/mcp-client'; +import {KEENABLE} from './keenable-mcp'; + +/** + * Keenable, through its MCP server, as Exa and Parallel are, signed in to or given an API key; its + * keyless server is the same service. Keenable says nothing of who signed in, so the person names + * each connection, and an organization can have as many as it likes. A new Keenable connection lets + * agents search the web and read pages. + */ +export const KEENABLE_MCP: BuiltInMcp = { + integrationId: 'keenable', + server: KEENABLE, + // Keenable turns a bad key down at the door, so listing the tools with it is enough. + apiKey: {check: async (key) => (await listMcpTools(KEENABLE, key)).map(() => undefined)}, + defaults: ['search', 'fetch'], + tools: { + search_web_pages: {actions: ['search'], run: (params) => ({collectionId: 'searches', operation: 'run', title: textParam(params, 'query')})}, + fetch_page_content: {actions: ['fetch'], run: (params) => ({collectionId: 'pages', operation: 'read', title: textParam(params, 'url')})}, + }, +}; diff --git a/packages/api/src/connections/take-over-custom.test.ts b/packages/api/src/connections/take-over-custom.test.ts index ad5a678d..538a3ab7 100644 --- a/packages/api/src/connections/take-over-custom.test.ts +++ b/packages/api/src/connections/take-over-custom.test.ts @@ -75,6 +75,7 @@ mock.module('../utils/secret-crypto', () => ({ const {movedCredential, movedPermissions, movedRun, takeOverCustomConnections} = await import('./take-over-custom'); const {CLOUDFLARE_MCP} = await import('./cloudflare/route'); const {EXA_MCP} = await import('./exa/route'); +const {KEENABLE_MCP} = await import('./keenable/route'); const {LINEAR_MCP} = await import('./linear/route'); const {PARALLEL_MCP} = await import('./parallel/route'); @@ -205,6 +206,16 @@ describe('movedCredential', () => { expect(movedCredential(EXA_MCP, {auth: 'none', url: 'https://mcp.exa.ai/mcp'})).toEqual({kind: 'stays'}); }); + test("Keenable's keyless server: a key typed in for it becomes the integration's API key, and reached with nothing it stays custom", () => { + expect(movedCredential(KEENABLE_MCP, {auth: 'token', url: 'https://api.keenable.ai/mcp', token: 'keen_a1_1234'})).toEqual({kind: 'moved', credential: {auth: 'key', key: 'keen_a1_1234'}}); + expect(movedCredential(KEENABLE_MCP, {auth: 'none', url: 'https://api.keenable.ai/mcp'})).toEqual({kind: 'stays'}); + expect(movedRun(KEENABLE_MCP, {id: 'e-1', action: 'fetch_page_content', query: JSON.stringify({url: 'https://example.com'})})).toMatchObject({ + collectionId: 'pages', + action: 'read', + recordTitle: 'https://example.com', + }); + }); + test('a sign-in through another authorization server has to be signed in to again', () => { expect(movedCredential(EXA_MCP, {...EXA_SIGN_IN, server: {...EXA_SIGN_IN.server, tokenUrl: 'https://elsewhere.example.com/token'}})).toEqual({kind: 'moved', credential: null}); expect(movedCredential(LINEAR_MCP, {...EXA_SIGN_IN, url: 'https://mcp.linear.app/sse'})).toEqual({kind: 'moved', credential: null}); diff --git a/packages/api/src/mcp/route.test.ts b/packages/api/src/mcp/route.test.ts index e4f2bdf0..59be03d2 100644 --- a/packages/api/src/mcp/route.test.ts +++ b/packages/api/src/mcp/route.test.ts @@ -122,7 +122,9 @@ describe('handleMcpRoute', () => { expect(result.instructions).toContain('with the task it was made for'); expect(result.instructions).toContain('- Email (alex@example.com), connection mail-1: Read, Archive'); expect(result.instructions).toContain('check Personal Agent Proxy first'); - expect(result.instructions).toContain('Personal Agent Proxy connects Email, Google, Granola, Notion, Linear, Cloudflare, Exa, Parallel, Plaid, PostgreSQL, MySQL and any custom MCP server'); + expect(result.instructions).toContain( + 'Personal Agent Proxy connects Email, Google, Granola, Notion, Linear, Cloudflare, Exa, Parallel, Keenable, Plaid, PostgreSQL, MySQL and any custom MCP server', + ); expect(result.instructions).toContain('https://app.example.com/org-1/connections/new'); expect(result.instructions).toContain('give them its url'); expect(result.instructions).toContain('send_feedback'); diff --git a/packages/api/src/records/connectors.ts b/packages/api/src/records/connectors.ts index e726990a..1b7afa54 100644 --- a/packages/api/src/records/connectors.ts +++ b/packages/api/src/records/connectors.ts @@ -10,7 +10,7 @@ import {notionConnector} from './notion-connector'; import {plaidConnector} from './plaid-connector'; import {sqlConnector} from './sql-connector'; import {vaultConnector} from './vault-connector'; -import {exaConnector, parallelConnector} from './web-connector'; +import {exaConnector, keenableConnector, parallelConnector} from './web-connector'; // A custom MCP server has no records: its tools are called as they are (`connections/custom`). const CONNECTORS: Record, Connector> = { @@ -22,6 +22,7 @@ const CONNECTORS: Record, Connector> = { cloudflare: cloudflareConnector, exa: exaConnector, parallel: parallelConnector, + keenable: keenableConnector, postgres: sqlConnector('postgres'), mysql: sqlConnector('mysql'), google: googleConnector, diff --git a/packages/api/src/records/web-connector.test.ts b/packages/api/src/records/web-connector.test.ts index 50b3939d..90350c66 100644 --- a/packages/api/src/records/web-connector.test.ts +++ b/packages/api/src/records/web-connector.test.ts @@ -3,7 +3,7 @@ import {Ok} from 'ts-results-es'; import {findCollection, findIntegration, type Collection} from '@proxy/integrations'; -// The servers' tools by name, answering as Exa's and Parallel's do. +// The servers' tools by name, answering as Exa's, Parallel's and Keenable's do. type Call = {server: string; name: string; args: Record}; const tools = {calls: [] as Call[], answer: (_call: Call): string => ''}; @@ -14,7 +14,9 @@ mock.module('../connections/mcp/mcp-session', () => ({ }, })); -const {exaConnector, parallelConnector} = await import('./web-connector'); +const {exaConnector, keenableConnector, parallelConnector} = await import('./web-connector'); + +const CONNECTORS = {exa: exaConnector, parallel: parallelConnector, keenable: keenableConnector}; function collection(integrationId: string, id: string): Collection { const integration = findIntegration(integrationId); @@ -25,7 +27,7 @@ function collection(integrationId: string, id: string): Collection { return found; } -function run(integrationId: 'exa' | 'parallel', command: 'run' | 'read', values: Record) { +function run(integrationId: keyof typeof CONNECTORS, command: 'run' | 'read', values: Record) { const target = { connection: { id: `${integrationId}-1`, @@ -42,7 +44,7 @@ function run(integrationId: 'exa' | 'parallel', command: 'run' | 'read', values: }, collection: collection(integrationId, command === 'run' ? 'searches' : 'pages'), }; - const runner = (integrationId === 'exa' ? exaConnector : parallelConnector).newCommands?.[command]; + const runner = CONNECTORS[integrationId].newCommands?.[command]; if (!runner) { throw new Error(`No ${command} for ${integrationId}`); } @@ -63,6 +65,15 @@ beforeEach(() => { if (name === 'web_search') { return JSON.stringify({results: [{url: 'https://bun.sh/docs/test/mocks', title: 'Mocks | Bun Docs', publish_date: null, excerpts: ['Mocking replaces a dependency.']}]}); } + if (name === 'search_web_pages') { + return JSON.stringify({ + query: 'bun module mocks', + results: [{url: 'https://bun.sh/docs/test/mocks', title: 'Mocks | Bun Docs', description: '', snippet: 'Mocking replaces a dependency.\n', acquired_at: '2026-10-13T09:00:00Z'}], + }); + } + if (name === 'fetch_page_content') { + return JSON.stringify({url: 'https://bun.sh/docs/test/mocks', title: 'Mocks | Bun Docs', content: 'Mocking replaces a dependency.', description: ''}); + } return JSON.stringify({results: [{url: 'https://bun.sh/docs/test/mocks', title: 'Mocks | Bun Docs', excerpts: [], full_content: 'x'.repeat(150_000)}], errors: []}); }; }); @@ -82,6 +93,20 @@ describe('webConnector', () => { expect(record?.values.results).toBe(EXA_RESULTS); }); + test('searches Keenable with the query, asking for JSON, its results written as Exa writes them', async () => { + const record = (await run('keenable', 'run', {query: 'bun module mocks'})).unwrap(); + + expect(tools.calls).toEqual([{server: 'Keenable', name: 'search_web_pages', args: {query: 'bun module mocks', output_format: 'json'}}]); + expect(record?.values.results).toBe(EXA_RESULTS); + }); + + test('reads a page with Keenable, as much of it as is kept', async () => { + const record = (await run('keenable', 'read', {url: 'https://bun.sh/docs/test/mocks'})).unwrap(); + + expect(tools.calls).toEqual([{server: 'Keenable', name: 'fetch_page_content', args: {url: 'https://bun.sh/docs/test/mocks', max_chars: 100_000, output_format: 'json'}}]); + expect(record?.values).toEqual({url: 'https://bun.sh/docs/test/mocks', title: 'Mocks | Bun Docs', content: 'Mocking replaces a dependency.'}); + }); + test('reads a page with Exa, as much of it as is kept', async () => { const record = (await run('exa', 'read', {url: 'https://bun.sh/docs/test/mocks'})).unwrap(); diff --git a/packages/api/src/records/web-connector.ts b/packages/api/src/records/web-connector.ts index 5c9b2b9c..e23457c3 100644 --- a/packages/api/src/records/web-connector.ts +++ b/packages/api/src/records/web-connector.ts @@ -3,16 +3,17 @@ import {Err, Ok, type Result} from 'ts-results-es'; import {ApiErr, type ApiError, Do} from '@proxy/utils'; import {EXA, parseExaPage} from '../connections/exa/exa-mcp'; +import {formatKeenableResults, KEENABLE, parseKeenablePage} from '../connections/keenable/keenable-mcp'; import type {McpServer} from '../connections/mcp/mcp-oauth'; import {callMcpToolFor} from '../connections/mcp/mcp-session'; import {formatParallelResults, parseParallelPage, PARALLEL} from '../connections/parallel/parallel-mcp'; import {onlyRun, type Connector, type DataRecord} from './connector'; /** - * The web, through a search engine made for agents, Exa or Parallel, fetched live from its MCP - * server with the connection's tokens. Searches and pages are only run, never listed, opened or - * kept: a search answers with the provider's results, written out the same way for both, a page - * with its title and text, the first MAX_PAGE_CHARACTERS of it. + * The web, through a search engine made for agents, Exa, Parallel or Keenable, fetched live from + * its MCP server with the connection's tokens. Searches and pages are only run, never listed, + * opened or kept: a search answers with the provider's results, written out the same way for each, + * a page with its title and text, the first MAX_PAGE_CHARACTERS of it. */ const MAX_PAGE_CHARACTERS = 100_000; @@ -42,6 +43,13 @@ const PARALLEL_WEB: WebProvider = { read: (url) => ({tool: 'web_fetch', args: {urls: [url], full_content: true}, answer: (text) => parseParallelPage(text, url)}), }; +// Keenable takes the query as it is, stops reading a page where it is cut, and answers both as JSON. +const KEENABLE_WEB: WebProvider = { + server: KEENABLE, + search: (query) => ({tool: 'search_web_pages', args: {query, output_format: 'json'}, answer: formatKeenableResults}), + read: (url) => ({tool: 'fetch_page_content', args: {url, max_chars: MAX_PAGE_CHARACTERS, output_format: 'json'}, answer: parseKeenablePage}), +}; + // An address a provider can read: `http` or `https`. function pageUrl(raw: string): Result { const trimmed = raw.trim(); @@ -80,3 +88,5 @@ function webConnector(provider: WebProvider): Connector { export const exaConnector = webConnector(EXA_WEB); export const parallelConnector = webConnector(PARALLEL_WEB); + +export const keenableConnector = webConnector(KEENABLE_WEB); diff --git a/packages/docs/content/connections/custom-mcp.mdx b/packages/docs/content/connections/custom-mcp.mdx index 4681e8a3..a7cd098f 100644 --- a/packages/docs/content/connections/custom-mcp.mdx +++ b/packages/docs/content/connections/custom-mcp.mdx @@ -27,12 +27,13 @@ authorization server to the same rule; a server on a private address needs Perso The servers of the services Personal Agent Proxy connects itself, [Granola](/connections/granola), [Notion](/connections/notion), [Linear](/connections/linear), -[Cloudflare](/connections/cloudflare) (`https://mcp.cloudflare.com/mcp`), [Exa](/connections/exa) -and [Parallel](/connections/parallel), are connected as those services, under the name you gave, -as adding them from the list of connections does: entering one's address opens the service's own -sign-in, or, for Cloudflare, Exa, Linear and Parallel with an access token typed in, or Exa's -address with its key in it (`?exaApiKey=`), connects the service with it as its API key. Granola and Notion are only -signed in to, so a token typed in for them is turned down. See also +[Cloudflare](/connections/cloudflare) (`https://mcp.cloudflare.com/mcp`), [Exa](/connections/exa), +[Parallel](/connections/parallel) and [Keenable](/connections/keenable), are connected as those +services, under the name you gave, as adding them from the list of connections does: entering +one's address opens the service's own sign-in, or, for Cloudflare, Exa, Keenable, Linear and +Parallel with an access token typed in, or Exa's address with its key in it (`?exaApiKey=`), +connects the service with it as its API key. Granola and Notion are only signed in to, so a token +typed in for them is turned down. See also [When a server becomes built in](#when-a-server-becomes-built-in). ### Sign in, if the server asks @@ -92,9 +93,9 @@ its agents and its activity: - Tool runs in the activity log become the service's own requests, such as listing Linear issues or running a web search. Runs of tools it has no action for stay as they were. - A server you signed in to stays signed in, and an access token typed in for Cloudflare, Exa, - Linear or Parallel, or an Exa key in the server's address, becomes the connection's API key, once - the service has accepted it. Any other token, one the service turns down, or a sign-in it has - since revoked, doesn't carry over: the connection has to be signed in to before agents can use - it, so open it and choose **Sign in**. -- A server you connected with no token or sign-in at all, as Parallel's free search server, stays a - custom server, so it keeps working as it did. + Keenable, Linear or Parallel, or an Exa key in the server's address, becomes the connection's + API key, once the service has accepted it. Any other token, one the service turns down, or a + sign-in it has since revoked, doesn't carry over: the connection has to be signed in to before + agents can use it, so open it and choose **Sign in**. +- A server you connected with no token or sign-in at all, as Parallel's or Keenable's free search + server, stays a custom server, so it keeps working as it did. diff --git a/packages/docs/content/connections/keenable.mdx b/packages/docs/content/connections/keenable.mdx new file mode 100644 index 00000000..d1ecdd85 --- /dev/null +++ b/packages/docs/content/connections/keenable.mdx @@ -0,0 +1,41 @@ +--- +title: Keenable +description: Connect Keenable for agents to search the web and read web pages. +--- + +Agents search the web and read pages through Keenable, on the Keenable account you sign in with or the +API key you give. What they search for and the pages they read go to Keenable, and searches count +toward that account's usage. Nothing is kept in Personal Agent Proxy beyond the activity log. + +## Connect Keenable + + + +### Sign in, or use an API key + +Open **Connections**, then **Add connection**. Choose **Keenable** to sign in with your Keenable account, +or the key beside it to paste an API key made at [app.keenable.ai](https://app.keenable.ai/console). The key is checked with +Keenable before the connection is made, and sent with every request as its bearer token. + +Keenable says nothing about who signed in, so a connection made with a key shows as the key's end, +and one signed in to as **Signed in**. Connect Keenable as often as you like, for another account or +another key; rename a connection from the list of connections to tell them apart. + +### Review the defaults + +A new Keenable connection gives every agent Search the web and Read web pages. Turn either off on the +connection's page, or for one agent on the agent's page. + + + +To sign in again, or replace the key, open the connection: under its permissions, choose **Sign in +again** or **use an API key**. The connection keeps its name and every agent's settings. + +## What agents can do + +- **Search the web** takes what to find, described in plain words, and answers with the pages that + fit, each with its title, address, publish date where known and the text that matches. +- **Read web pages** opens one page by its address and answers with its title and text as Markdown, + the first 100,000 characters of it. + +Agents can't change anything through Keenable: both actions only read. diff --git a/packages/docs/content/index.mdx b/packages/docs/content/index.mdx index d97f1a43..e0830376 100644 --- a/packages/docs/content/index.mdx +++ b/packages/docs/content/index.mdx @@ -19,7 +19,8 @@ for what agents should know about you, [Email](/connections/email) for IMAP and [Granola](/connections/granola) for meeting notes and transcripts, [Notion](/connections/notion) for pages and databases, [Linear](/connections/linear) for issues, [Cloudflare](/connections/cloudflare) for zones, DNS, Workers and the rest of an account, -[Exa](/connections/exa) or [Parallel](/connections/parallel) for searching and reading the web, +[Exa](/connections/exa), [Parallel](/connections/parallel) or [Keenable](/connections/keenable) +for searching and reading the web, [Plaid](/connections/plaid) for bank balances and transactions, and [PostgreSQL](/connections/postgres) or [MySQL](/connections/mysql) for a database's tables and queries. [Custom MCP servers](/connections/custom-mcp) explains how to connect any other service diff --git a/packages/docs/src/nav.ts b/packages/docs/src/nav.ts index 6d30929f..02a81942 100644 --- a/packages/docs/src/nav.ts +++ b/packages/docs/src/nav.ts @@ -27,6 +27,7 @@ export const NAV: {title: string; pages: NavEntry[]}[] = [ 'connections/cloudflare', 'connections/exa', 'connections/parallel', + 'connections/keenable', 'connections/plaid', 'connections/postgres', 'connections/mysql', diff --git a/packages/integrations/src/catalog.ts b/packages/integrations/src/catalog.ts index 78e7ec95..fee54fa0 100644 --- a/packages/integrations/src/catalog.ts +++ b/packages/integrations/src/catalog.ts @@ -641,7 +641,7 @@ const cloudflare: Integration = { // the provider's results, each page's title, address and the passages that fit the query; a page as // its text. Nothing is listed or kept: searches and pages are only ever run. function webIntegration( - id: 'exa' | 'parallel', + id: 'exa' | 'parallel' | 'keenable', name: string, description: string, keysUrl: string, @@ -737,6 +737,13 @@ const parallel = webIntegration( 'https://platform.parallel.ai', ); +const keenable = webIntegration( + 'keenable', + 'Keenable', + 'Search and read the web', + 'https://app.keenable.ai/console', +); + // A SQL database, signed in to with a database user's name and password: PostgreSQL, or MySQL // and MariaDB. What the user may do in the database is the limit, so access is narrowed in // Personal Agent Proxy: the tables and their rows are one tree, a table holding its rows, and a @@ -1553,6 +1560,7 @@ export const INTEGRATIONS: Integration[] = [ cloudflare, exa, parallel, + keenable, plaid, postgres, mysql, diff --git a/packages/integrations/src/tools.test.ts b/packages/integrations/src/tools.test.ts index 39b6ce41..144ee25a 100644 --- a/packages/integrations/src/tools.test.ts +++ b/packages/integrations/src/tools.test.ts @@ -210,12 +210,13 @@ describe('postgres and mysql', () => { }); }); -describe('exa and parallel', () => { +describe('exa, parallel and keenable', () => { const exa = integration('exa'); test('searches and pages only ever run, each a read with an action of its own', () => { expect(integrationTools(exa).map((tool) => tool.name)).toEqual(['searches_run', 'pages_read']); expect(integrationTools(integration('parallel'))).toEqual(integrationTools(exa)); + expect(integrationTools(integration('keenable'))).toEqual(integrationTools(exa)); expect(findTool(exa, 'searches_run')).toMatchObject({ readOnly: true, action: 'search', diff --git a/packages/integrations/src/types.ts b/packages/integrations/src/types.ts index 9814262d..e2350e43 100644 --- a/packages/integrations/src/types.ts +++ b/packages/integrations/src/types.ts @@ -108,6 +108,7 @@ export type IntegrationId = | 'cloudflare' | 'exa' | 'parallel' + | 'keenable' | 'postgres' | 'mysql' | 'mcp' @@ -136,10 +137,10 @@ export type Integration = { // `/api/connections//start`: Google, and every integration built on its MCP server. signIn?: true; // Also connected with an API key, made at `keysUrl`, on `/api/connections/`: Cloudflare, - // Exa, Linear and Parallel, whose MCP servers take one as a bearer token. + // Exa, Keenable, Linear and Parallel, whose MCP servers take one as a bearer token. apiKey?: {keysUrl: string}; - // Says nothing of who signed in, or whose a key is: Exa and Parallel. Each sign-in or key is a - // connection of its own, which only its own page signs in to again or gives a new key. + // Says nothing of who signed in, or whose a key is: Exa, Keenable and Parallel. Each sign-in or + // key is a connection of its own, which only its own page signs in to again or gives a new key. anonymous?: true; // The experiment an organization needs on to connect it: the catalog, the api's routes and the // MCP server's instructions all leave it out without. Connections already made keep working. diff --git a/packages/web/public/integrations/keenable.svg b/packages/web/public/integrations/keenable.svg new file mode 100644 index 00000000..66fd7e4e --- /dev/null +++ b/packages/web/public/integrations/keenable.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/packages/web/src/client/connections-client.ts b/packages/web/src/client/connections-client.ts index 553d6139..97770f71 100644 --- a/packages/web/src/client/connections-client.ts +++ b/packages/web/src/client/connections-client.ts @@ -123,10 +123,10 @@ export function connectMcp(org: string, input: ConnectMcpInput) { export type ConnectWithKeyInput = {key: string; connection?: string; shared?: boolean}; /** - * Connects an integration that takes an API key, Cloudflare, Exa, Linear or Parallel, with one, - * checked with the service first: a new connection, or, with `connection`, that connection given - * the key, its name kept. 422 is a key the service turned down, 502 the service out of reach, 400 - * says what is wrong with the key. + * Connects an integration that takes an API key, Cloudflare, Exa, Keenable, Linear or Parallel, + * with one, checked with the service first: a new connection, or, with `connection`, that + * connection given the key, its name kept. 422 is a key the service turned down, 502 the service + * out of reach, 400 says what is wrong with the key. */ export function connectWithKey( org: string, diff --git a/packages/web/src/components/connect-with-key-dialog.tsx b/packages/web/src/components/connect-with-key-dialog.tsx index 93c88dcf..e00173ce 100644 --- a/packages/web/src/components/connect-with-key-dialog.tsx +++ b/packages/web/src/components/connect-with-key-dialog.tsx @@ -41,10 +41,10 @@ type Props = { onConnected: (connection: Connection) => void; }; -// Connecting an integration that takes an API key, Cloudflare, Exa, Linear or Parallel, in a small -// dialog: the key, checked with the service before the connection is made, and where to get one. -// The connection shows as the key's account, or its end where the service names none; it can be -// renamed from the connections list. +// Connecting an integration that takes an API key, Cloudflare, Exa, Keenable, Linear or Parallel, +// in a small dialog: the key, checked with the service before the connection is made, and where to +// get one. The connection shows as the key's account, or its end where the service names none; it +// can be renamed from the connections list. export function ConnectWithKeyDialog({ integrationId, connectionId, diff --git a/packages/web/src/components/integration-catalog.tsx b/packages/web/src/components/integration-catalog.tsx index 6cd3a4e7..5c8e6b19 100644 --- a/packages/web/src/components/integration-catalog.tsx +++ b/packages/web/src/components/integration-catalog.tsx @@ -23,9 +23,9 @@ import {useOrgSlug} from '@/hooks/use-org'; // The integrations not connected by signing in to them are connected on a form of their own: a // mailbox with an app password, a database with a user's name and password, a custom MCP server by -// its address. One that also takes an API key, as Cloudflare, Exa, Linear and Parallel do, has a -// key before its plus, which takes the key in a dialog here; its plus is then a link of its own -// beside the row's. +// its address. One that also takes an API key, as Cloudflare, Exa, Keenable, Linear and Parallel +// do, has a key before its plus, which takes the key in a dialog here; its plus is then a link of +// its own beside the row's. // The whole row is the link or button, out in the list's gutter as a row of `RowList` is; its plus is // styled as a ghost icon button that lights up with the row, and dimmed on a row greyed out. From afe17260f6ec7baaa554e8166250ae4139c67939 Mon Sep 17 00:00:00 2001 From: Ilya Bogin Date: Sat, 31 Oct 2026 11:05:53 +0200 Subject: [PATCH 2/2] port keenable to the catalog architecture of #167 A page Keenable found nothing at is not found, and one whose site turned its fetcher away for too many requests is out of reach for now, each with the address and what to try next, as Exa's and Parallel's are since #165; one it couldn't reach is out of reach in its own words, and anything else stays in its words. As Exa is, it is asked for a little more of a page than is kept, so a page cut at 100,000 characters ends with the note saying so, and an empty page fails. Custom connections to api.keenable.ai move over on the next start through a data migration of their own, since instances have already run the one that moved the others'. --- .../connections/keenable/keenable-mcp.test.ts | 37 +++++++++++++++- .../src/connections/keenable/keenable-mcp.ts | 44 ++++++++++++++++--- packages/api/src/connections/page-failure.ts | 10 ++--- packages/api/src/data-migrations/run.ts | 2 + .../api/src/records/web-connector.test.ts | 34 +++++++++++--- .../docs/content/connections/keenable.mdx | 4 +- 6 files changed, 114 insertions(+), 17 deletions(-) diff --git a/packages/api/src/connections/keenable/keenable-mcp.test.ts b/packages/api/src/connections/keenable/keenable-mcp.test.ts index 60c61c15..b8993dd4 100644 --- a/packages/api/src/connections/keenable/keenable-mcp.test.ts +++ b/packages/api/src/connections/keenable/keenable-mcp.test.ts @@ -1,6 +1,8 @@ import {describe, expect, test} from 'bun:test'; -import {formatKeenableResults, parseKeenablePage} from './keenable-mcp'; +import {ApiErr} from '@proxy/utils'; + +import {formatKeenableResults, KEENABLE, parseKeenablePage} from './keenable-mcp'; describe('formatKeenableResults', () => { test('writes the results out as Exa writes its own, a block a page, its snippet as the highlights', () => { @@ -41,3 +43,36 @@ describe('parseKeenablePage', () => { expect(parseKeenablePage(JSON.stringify({url: 'https://example.com/', content: 'Just text\n'})).unwrap()).toEqual({title: '', content: 'Just text'}); }); }); + +describe('KEENABLE.toolError', () => { + const toolError = (text: string) => KEENABLE.toolError?.(text); + const page = (url: string, reason: string) => toolError(`Error fetching ${url}: ${reason}`); + + test('a page Keenable found nothing at is not found, with its address and what to do next, as Exa’s is', () => { + expect(page('https://example.com/docs/old-guide', 'The page does not exist at this URL. Check that the URL is correct.')).toMatchObject({ + kind: 'not_found', + resource: 'page', + id: 'https://example.com/docs/old-guide', + message: expect.stringContaining('Keenable found no page at https://example.com/docs/old-guide'), + }); + }); + + test('a site that turned its fetcher away for too many requests, or that it couldn’t reach, is out of reach for now', () => { + expect(page('https://example.com/busy', 'The target server rate-limited this fetch, please retry shortly.')).toMatchObject({ + kind: 'provider_unreachable', + cause: {kind: 'http', status: 429}, + message: expect.stringContaining('too many requests'), + }); + expect(page('https://example.com/down', 'The target page could not be reached by the fetcher (upstream fetch failed). This may be transient — retrying is reasonable.')).toMatchObject({ + kind: 'provider_unreachable', + message: 'Keenable couldn’t read https://example.com/down: The target page could not be reached by the fetcher (upstream fetch failed). This may be transient — retrying is reasonable.', + }); + }); + + test('any other failure is in Keenable’s words', () => { + expect(page('https://example.com/members', 'The target server denied access to this URL')).toEqual( + ApiErr.validationError('Keenable turned it down: Error fetching https://example.com/members: The target server denied access to this URL'), + ); + expect(toolError('Invalid arguments')).toEqual(ApiErr.validationError('Keenable turned it down: Invalid arguments')); + }); +}); diff --git a/packages/api/src/connections/keenable/keenable-mcp.ts b/packages/api/src/connections/keenable/keenable-mcp.ts index 56409139..0631d0d7 100644 --- a/packages/api/src/connections/keenable/keenable-mcp.ts +++ b/packages/api/src/connections/keenable/keenable-mcp.ts @@ -5,15 +5,17 @@ import {ApiErr, type ApiError} from '@proxy/utils'; import {parseToolJson} from '../mcp/mcp-client'; import type {McpServer} from '../mcp/mcp-oauth'; +import {httpPageFailure, pageError, type PageFailure} from '../page-failure'; /** * Keenable's MCP server, at the address that takes a sign-in or an API key (its keyless one is * `/mcp` beside it), and what its tools answer when asked for JSON: a search's results each a * page's address, title, publish date where known and a snippet of its text, and a page read its - * title and text as Markdown. A page it couldn't read fails the tool, saying why in words: "Error - * fetching https://…: The page does not exist at this URL.". Results are written out as Exa writes - * its own, so an agent reads them all the same. Its sign-in is Keenable's Supabase authorization - * server, as the server's protected resource metadata names it. + * title and text as Markdown, cut where it reaches `max_chars` with a note of its own after it. A + * page it couldn't read fails the tool, saying why in words: "Error fetching https://…: The page + * does not exist at this URL. Check that the URL is correct.". Results are written out as Exa + * writes its own, so an agent reads them all the same. Its sign-in is Keenable's Supabase + * authorization server, as the server's protected resource metadata names it. */ const SERVER = 'https://api.keenable.ai/mcp-oauth'; @@ -31,9 +33,41 @@ export const KEENABLE: McpServer = { tokenUrl: `${AUTH_BASE}/oauth/token`, // Supabase's own default; Keenable asks for nothing more. scope: 'email', - toolError: (text) => ApiErr.validationError(`Keenable turned it down: ${text.trim() || 'it didn’t say why'}`), + toolError: keenableToolError, }; +// "Error fetching https://example.com/gone: The page does not exist at this URL. Check that the URL +// is correct.", for the one page read. +const FETCH_ERROR = /^Error fetching (\S+): (.+)$/; + +// What Keenable's words say of a page, where they name a failure of a known kind: nothing at the +// address, or the site turning its fetcher away for too many requests. Others, as a site that denied +// it access without saying how, are passed on in its words. +function fetchFailure(reason: string): PageFailure | null { + if (/does not exist at this URL/i.test(reason)) { + return {kind: 'missing'}; + } + if (/rate-limited/i.test(reason)) { + return httpPageFailure(429); + } + return null; +} + +// A page Keenable couldn't read, by what it says of it, with the address; one it couldn't reach, +// which it says may be worth trying again, out of reach in its words; anything else in its words. +function keenableToolError(text: string): ApiError { + const fetchError = FETCH_ERROR.exec(text.trim()); + const [url, reason] = [fetchError?.[1], fetchError?.[2]?.trim()]; + const failure = reason ? fetchFailure(reason) : null; + if (url && failure) { + return pageError('Keenable', url, failure); + } + if (url && reason && /may be transient/i.test(reason)) { + return ApiErr.providerUnreachable(new Error(`Keenable couldn’t reach ${url}`), `Keenable couldn’t read ${url}: ${reason}`); + } + return ApiErr.validationError(`Keenable turned it down: ${text.trim() || 'it didn’t say why'}`); +} + const resultSchema = z.object({ url: z.string(), title: z.string().nullish(), diff --git a/packages/api/src/connections/page-failure.ts b/packages/api/src/connections/page-failure.ts index 358fc5d6..e227fedc 100644 --- a/packages/api/src/connections/page-failure.ts +++ b/packages/api/src/connections/page-failure.ts @@ -1,11 +1,11 @@ import {ApiErr, type ApiError, type FetchError} from '@proxy/utils'; /** - * Why a web page couldn't be read, from what Exa or Parallel says in its own way, as an error an - * agent can act on: the address it asked for, and whether to look for it elsewhere or try again. - * Nothing at the address is not found; a site that was slow, busy or failing, out of reach for - * now; a site that turned the crawler away, or a page with no text or that isn't text, turned - * down, there being no point in trying again. + * Why a web page couldn't be read, from what Exa, Parallel or Keenable says in its own way, as an + * error an agent can act on: the address it asked for, and whether to look for it elsewhere or + * try again. Nothing at the address is not found; a site that was slow, busy or failing, out of + * reach for now; a site that turned the crawler away, or a page with no text or that isn't text, + * turned down, there being no point in trying again. */ export type PageFailure = {kind: 'missing' | 'timeout' | 'unsupported' | 'empty'} | {kind: 'refused' | 'busy'; status: number}; diff --git a/packages/api/src/data-migrations/run.ts b/packages/api/src/data-migrations/run.ts index e5268131..9df5f326 100644 --- a/packages/api/src/data-migrations/run.ts +++ b/packages/api/src/data-migrations/run.ts @@ -22,6 +22,8 @@ export type DataMigration = {id: string; run: () => Promise { expect(record?.values).toEqual({url: 'https://bun.sh/docs/test/mocks', title: 'Mocks | Bun Docs', content: 'Mocking replaces a dependency.'}); }); + test('a page Keenable cut is cut where it is kept, its own note on the cut dropped for the one saying the page goes on', async () => { + const text = 'Section one. '.repeat(10_000).slice(0, 101_000); + tools.answer = () => + JSON.stringify({ + url: 'https://docs.example.com/reference', + title: 'Reference | Example Docs', + content: `${text}\n\n..._This content has been truncated to stay below 101000 characters_...`, + description: '', + }); + const content = (await run('keenable', 'read', {url: 'https://docs.example.com/reference'})).unwrap()?.values.content ?? ''; + + expect(content.startsWith(text.slice(0, 100_000))).toBe(true); + expect(content.slice(100_000)).toStartWith('\n\n[Cut short: the page goes on past 100,000 characters, of which these are the first 100,000.'); + }); + test('reads a page with Exa, as much of it as is kept', async () => { const record = (await run('exa', 'read', {url: 'https://bun.sh/docs/test/mocks'})).unwrap(); @@ -138,11 +153,16 @@ describe('webConnector', () => { expect(content.slice(100_000)).toStartWith('\n\n[Cut short: the page is 150,000 characters long, of which these are the first 100,000.'); }); - test('a page with no text is a failure saying so, for Exa and Parallel alike', async () => { - tools.answer = ({name}) => - name === 'web_fetch_exa' - ? 'No content found for the provided URL(s).' - : JSON.stringify({results: [{url: 'https://app.example.com', title: 'App', excerpts: [' '], full_content: ' '}], errors: []}); + test('a page with no text is a failure saying so, for Exa, Parallel and Keenable alike', async () => { + tools.answer = ({name}) => { + if (name === 'web_fetch_exa') { + return 'No content found for the provided URL(s).'; + } + if (name === 'fetch_page_content') { + return JSON.stringify({url: 'https://app.example.com', title: 'App', content: ' ', description: ''}); + } + return JSON.stringify({results: [{url: 'https://app.example.com', title: 'App', excerpts: [' '], full_content: ' '}], errors: []}); + }; expect((await run('exa', 'read', {url: 'https://app.example.com'})).unwrapErr()).toMatchObject({ kind: 'validation_error', @@ -152,6 +172,10 @@ describe('webConnector', () => { kind: 'validation_error', message: expect.stringContaining('Parallel found no text at https://app.example.com'), }); + expect((await run('keenable', 'read', {url: 'https://app.example.com'})).unwrapErr()).toMatchObject({ + kind: 'validation_error', + message: expect.stringContaining('Keenable found no text at https://app.example.com'), + }); }); test('refuses an empty query and an address that is no web page, before asking the provider', async () => { diff --git a/packages/docs/content/connections/keenable.mdx b/packages/docs/content/connections/keenable.mdx index d1ecdd85..071db340 100644 --- a/packages/docs/content/connections/keenable.mdx +++ b/packages/docs/content/connections/keenable.mdx @@ -36,6 +36,8 @@ again** or **use an API key**. The connection keeps its name and every agent's s - **Search the web** takes what to find, described in plain words, and answers with the pages that fit, each with its title, address, publish date where known and the text that matches. - **Read web pages** opens one page by its address and answers with its title and text as Markdown, - the first 100,000 characters of it. + the first 100,000 characters of it, followed by a note where the page goes on. A page that can't + be read fails, saying why and what to try next: nothing at the address, the site turned the + fetcher away, no text to read, or the site out of reach or busy for now. Agents can't change anything through Keenable: both actions only read.