improvement(api): publish an RFC 9727 API catalog and stop padding tables in docs markdown (#9066)

* improvement(api): publish an RFC 9727 API catalog and compact indented tables in docs markdown

* fix(docs): turn off table padding in the markdown serializer and assert exact catalog URLs

Author: waleedlatif1

apps/docs/lib/llms.ts

@@ -4,32 +4,6 @@ import { getIntegrationLandingPage, INTEGRATION_LANDING_LABEL } from '@/lib/inte
 import { getApiSpecContent } from '@/lib/openapi'
 import type { PageData, source } from '@/lib/source'
 
-const TABLE_ROW = /^\|.*\|$/
-const TABLE_DELIMITER_CELL = /^\s*(:?)-+(:?)\s*$/
-const UNESCAPED_PIPE = /(?<!\\)\|/
-
-/**
- * The processed-markdown serializer pads every table cell to its column's widest cell, so one long
- * enum (a parameter listing every timezone) pads every row of its table and roughly doubles the
- * integration reference. Drop that trailing padding; a cell's own leading indentation (the `↳`
- * nesting in output tables) and inline code are kept, and the tables stay valid GFM.
- */
-function compactTables(markdown: string): string {
-  let inFence = false
-  return markdown
-    .split('\n')
-    .map((line) => {
-      if (line.startsWith('```')) inFence = !inFence
-      if (inFence || !TABLE_ROW.test(line)) return line
-      const cells = line.slice(1, -1).split(UNESCAPED_PIPE)
-      if (cells.every((cell) => TABLE_DELIMITER_CELL.test(cell))) {
-        return `|${cells.map((cell) => cell.replace(TABLE_DELIMITER_CELL, ' $1---$2 ')).join('|')}|`
-      }
-      return `|${cells.map((cell) => ` ${cell.replace(/^ /, '').trimEnd()} `).join('|')}|`
-    })
-    .join('\n')
-}
-
 type LLMPageData = PageData & { getAPIPageProps?: () => ApiPageProps }
 
 /**
@@ -62,5 +36,5 @@ export async function getLLMText(page: InferPageType<typeof source>) {
     : ''
   return `${heading}
 
-${landingLink}${compactTables(processed)}`
+${landingLink}${processed}`
 }

apps/docs/source.config.ts

@@ -1,5 +1,6 @@
 import { execFileSync } from 'node:child_process'
 import path from 'node:path'
+import { remarkGfm } from 'fumadocs-core/mdx-plugins'
 import { defineConfig, defineDocs, frontmatterSchema } from 'fumadocs-mdx/config'
 import lastModified from 'fumadocs-mdx/plugins/last-modified'
 import { curlJsonBodyGrammar } from './lib/shiki-curl-json'
@@ -66,6 +67,22 @@ export default defineConfig({
     lastModified({ versionControl: async (file) => gitLastModified?.get(path.resolve(file)) }),
   ],
   mdxOptions: {
+    /**
+     * The processed markdown behind `.md` pages, `llms-full.txt`, and Copy page is serialized by
+     * remark-gfm, which by default pads every table cell to its column's widest cell — one long
+     * enum pads a whole table. `tablePipeAlign: false` keeps the table and drops the padding; it
+     * only affects serializing to markdown, never the rendered page.
+     */
+    remarkPlugins: (plugins) => {
+      if (!plugins.includes(remarkGfm)) {
+        throw new Error(
+          'The fumadocs-mdx preset no longer registers remarkGfm; reapply tablePipeAlign'
+        )
+      }
+      return plugins.map((plugin) =>
+        plugin === remarkGfm ? [remarkGfm, { tablePipeAlign: false }] : plugin
+      )
+    },
     /**
      * Shiki defaults to `github-light` / `github-dark`, whose blues and purples appear nowhere
      * in the product. These themes carry the platform's own token colors instead — see

apps/sim/app/.well-known/api-catalog/route.ts

@@ -0,0 +1,43 @@
+import { NextResponse } from 'next/server'
+import { getBaseUrl } from '@/lib/core/utils/urls'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { DOCS_URL } from '@/lib/help-links'
+
+/** RFC 9727 §4.2: the catalog is a Linkset carrying the RFC's profile. */
+const API_CATALOG_CONTENT_TYPE =
+  'application/linkset+json; profile="https://www.rfc-editor.org/info/rfc9727"'
+
+/**
+ * RFC 9727 API catalog: one machine-readable entry point to the v2 REST API, linking its OpenAPI
+ * document (`service-desc`), reference docs (`service-doc`), and health endpoint (`status`) with
+ * the RFC 8631 relations. The anchor is this deployment's own origin, so a self-hosted instance
+ * describes itself. The OpenAPI document is the one published contract; its `servers` names Sim
+ * Cloud, so a client of another deployment uses the anchor as its base URL, as the API docs say.
+ * `Link: rel="api-catalog"` makes a HEAD request discoverable too (RFC 9727 §2).
+ */
+export const GET = withRouteHandler(() => {
+  const baseUrl = getBaseUrl()
+  const catalogUrl = `${baseUrl}/.well-known/api-catalog`
+
+  return NextResponse.json(
+    {
+      linkset: [
+        {
+          anchor: `${baseUrl}/api/v2`,
+          'service-desc': [
+            { href: `${DOCS_URL}/openapi.json`, type: 'application/vnd.oai.openapi+json' },
+          ],
+          'service-doc': [{ href: `${DOCS_URL}/api-reference/getting-started`, type: 'text/html' }],
+          status: [{ href: `${baseUrl}/api/health`, type: 'application/json' }],
+        },
+      ],
+    },
+    {
+      headers: {
+        'Content-Type': API_CATALOG_CONTENT_TYPE,
+        'Cache-Control': 'public, max-age=86400',
+        Link: `<${catalogUrl}>; rel="api-catalog"`,
+      },
+    }
+  )
+})

apps/sim/proxy.ts

@@ -295,15 +295,16 @@ function handleSecurityFiltering(request: NextRequest): NextResponse | null {
     pathname.startsWith('/api/webhooks/tiktok') ||
     pathname.startsWith('/api/webhooks/agentmail')
   const isMcpEndpoint = pathname.startsWith('/api/mcp/')
-  const isMcpDiscoveryEndpoint =
+  const isDiscoveryEndpoint =
     pathname.startsWith('/.well-known/oauth-authorization-server') ||
     pathname.startsWith('/.well-known/oauth-protected-resource') ||
-    pathname === '/.well-known/openai-apps-challenge'
+    pathname === '/.well-known/openai-apps-challenge' ||
+    pathname === '/.well-known/api-catalog'
   const isSuspicious = SUSPICIOUS_UA_PATTERNS.some((pattern) => pattern.test(userAgent))
 
   // Block suspicious requests, but exempt machine-to-machine endpoints that may
-  // legitimately omit User-Agent headers (webhooks and MCP protocol discovery/calls).
-  if (isSuspicious && !isWebhookEndpoint && !isMcpEndpoint && !isMcpDiscoveryEndpoint) {
+  // legitimately omit User-Agent headers (webhooks, MCP calls, and protocol discovery).
+  if (isSuspicious && !isWebhookEndpoint && !isMcpEndpoint && !isDiscoveryEndpoint) {
     // Scanner traffic (empty or tool user agents probing /admin.php etc.); the 403 is the response.
     logger.debug('Blocked suspicious request', {
       userAgent,

apps/sim/scripts/test-mcp-host-e2e.ts

@@ -3,16 +3,20 @@ import { mkdir, writeFile } from 'node:fs/promises'
 import { dirname } from 'node:path'
 import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
+import { SIM_DOCS_URL } from '@sim/utils/site'
 
 /**
- * Exercises the public ownership challenge, host isolation, and the app-wide JSON 404 through a
- * running local app.
+ * Exercises the public ownership challenge, host isolation, the RFC 9727 API catalog, and the
+ * app-wide JSON 404 through a running local app.
  */
 const logger = createLogger('McpHostE2E')
 const CHALLENGE_PATH = '/.well-known/openai-apps-challenge'
 const EXPECTED_CHALLENGE = 'lFJ1-XIWpHGRNcgzTPl2Y_yYCTWLvlIbAvNRD08EvLI'
 const MCP_HOST = 'mcp.sim.test'
 const PROTECTED_RESOURCE_PATH = '/.well-known/oauth-protected-resource'
+const API_CATALOG_PATH = '/.well-known/api-catalog'
+const API_CATALOG_CONTENT_TYPE =
+  'application/linkset+json; profile="https://www.rfc-editor.org/info/rfc9727"'
 const startedAt = new Date().toISOString()
 
 const configuredBaseUrl = process.env.MCP_HOST_E2E_BASE_URL
@@ -83,7 +87,7 @@ try {
     })
   }
 
-  for (const path of [`${CHALLENGE_PATH}/extra`, '/login', '/api/health']) {
+  for (const path of [`${CHALLENGE_PATH}/extra`, '/login', '/api/health', API_CATALOG_PATH]) {
     await check(`dedicated MCP host rejects ${path}`, async () => {
       const { response } = await request(path, 'Mozilla/5.0')
       assert.equal(response.status, 404)
@@ -109,6 +113,31 @@ try {
     })
   }
 
+  for (const userAgent of ['', 'python-requests/2.32.3']) {
+    await check(`API catalog for ${userAgent || 'an empty User-Agent'}`, async () => {
+      const { response, body } = await request(API_CATALOG_PATH, userAgent, baseUrl.host)
+      assert.equal(response.status, 200)
+      assert.equal(response.headers.get('content-type'), API_CATALOG_CONTENT_TYPE)
+      assert.equal(
+        response.headers.get('link'),
+        `<${baseUrl.origin}${API_CATALOG_PATH}>; rel="api-catalog"`
+      )
+      const [api] = JSON.parse(body).linkset
+      assert.equal(api.anchor, `${baseUrl.origin}/api/v2`)
+      assert.equal(api['service-desc'][0].href, `${SIM_DOCS_URL}/openapi.json`)
+      assert.equal(api.status[0].href, `${baseUrl.origin}/api/health`)
+      const health = await request(new URL(api.status[0].href).pathname, userAgent, baseUrl.host)
+      assert.equal(health.response.status, 200, 'The catalog status link must resolve')
+    })
+  }
+
+  await check('API catalog answers HEAD with its Link header (RFC 9727 §2)', async () => {
+    const { response, body } = await request(API_CATALOG_PATH, '', baseUrl.host, 'HEAD')
+    assert.equal(response.status, 200)
+    assert.equal(body, '')
+    assert.match(response.headers.get('link') ?? '', /rel="api-catalog"$/)
+  })
+
   await check('application host still serves health checks', async () => {
     const { response } = await request('/api/health', 'Mozilla/5.0', baseUrl.host)
     assert.equal(response.status, 200)