fix(docs): publish full LLM text as a static asset (#8769)

* fix(docs): publish full LLM text as a static asset

* fix(docs): regenerate full text during development

Author: waleedlatif1

apps/docs/.gitignore

@@ -41,3 +41,7 @@ next-env.d.ts
 
 # fumadocs generates .source dirs anywhere a source.config sits
 **/.source/
+
+# Build-generated documentation asset
+/public/llms-full.txt
+/public/.llms-full-*/

apps/docs/app/llms-full.txt/route.ts

@@ -16,6 +16,17 @@ const config: NextConfig = {
     webpackMemoryOptimizations: true,
     webpackBuildWorker: true,
   },
+  async headers() {
+    return [
+      {
+        source: '/llms-full.txt',
+        headers: [
+          { key: 'Content-Type', value: 'text/plain; charset=utf-8' },
+          { key: 'Cache-Control', value: 'public, max-age=0, s-maxage=86400, must-revalidate' },
+        ],
+      },
+    ]
+  },
   async redirects() {
     return DOCS_REDIRECTS
   },

apps/docs/next.config.ts

@@ -4,18 +4,19 @@
   "private": true,
   "license": "Apache-2.0",
   "scripts": {
-    "dev": "bun run dev:cache:cap && next dev --port 3001",
+    "dev": "bun run dev:cache:cap && bun run llms-full:generate && bunx concurrently --kill-others --names Next,LLMs \"next dev --port 3001\" \"bun scripts/watch-llms-full.ts\"",
     "dev:cache:cap": "bun run ../../scripts/prune-turbopack-cache.ts",
     "dev:clean": "rm -rf .next/dev/cache",
-    "build": "fumadocs-mdx && NODE_OPTIONS='--max-old-space-size=8192' next build",
+    "build": "bun run llms-full:generate && NODE_OPTIONS='--max-old-space-size=8192' next build",
     "start": "next start",
     "postinstall": "fumadocs-mdx",
     "test": "vitest run",
     "type-check": "fumadocs-mdx && tsc --noEmit",
     "lint": "biome check --write --unsafe .",
     "lint:check": "biome check .",
     "format": "biome format --write .",
-    "format:check": "biome format ."
+    "format:check": "biome format .",
+    "llms-full:generate": "fumadocs-mdx && bun scripts/generate-llms-full.ts"
   },
   "dependencies": {
     "@sim/db": "workspace:*",

apps/docs/package.json

@@ -0,0 +1,47 @@
+import { createWriteStream } from 'node:fs'
+import { mkdtemp, rename, rm } from 'node:fs/promises'
+import path from 'node:path'
+import { Readable } from 'node:stream'
+import { pipeline } from 'node:stream/promises'
+import { createLogger } from '@sim/logger'
+import { compareStrings } from '@sim/utils/string'
+import { plugin } from 'bun'
+import { createMdxPlugin } from 'fumadocs-mdx/bun'
+
+const MAX_LLM_TEXT_BYTES = 64 * 1024 * 1024
+const logger = createLogger('DocsLlmsFullAsset')
+
+await plugin(createMdxPlugin({ disableMetaFile: true }))
+
+const { source } = await import('@/lib/source')
+const { getLLMText } = await import('@/lib/llms')
+const publicDirectory = path.resolve('public')
+const temporaryDirectory = await mkdtemp(path.join(publicDirectory, '.llms-full-'))
+const temporaryFile = path.join(temporaryDirectory, 'llms-full.txt')
+
+async function* generateText(): AsyncGenerator<string> {
+  let hasContent = false
+  let bytes = 0
+  const pages = [...source.getPages()].sort((left, right) => compareStrings(left.url, right.url))
+  for (const page of pages) {
+    if (!page?.data || !page.url) continue
+    const text = await getLLMText(page)
+    if (!text) continue
+    const chunk = `${hasContent ? '\n\n---\n\n' : ''}${text}`
+    bytes += Buffer.byteLength(chunk)
+    if (bytes > MAX_LLM_TEXT_BYTES) {
+      throw new Error('Full documentation text exceeds the static asset size budget')
+    }
+    yield chunk
+    hasContent = true
+  }
+  if (!hasContent) throw new Error('No documentation text was generated')
+}
+
+try {
+  await pipeline(Readable.from(generateText()), createWriteStream(temporaryFile))
+  await rename(temporaryFile, path.join(publicDirectory, 'llms-full.txt'))
+  logger.info('Generated full documentation static asset')
+} finally {
+  await rm(temporaryDirectory, { recursive: true, force: true })
+}

apps/docs/scripts/generate-llms-full.ts

@@ -0,0 +1,51 @@
+import { watch } from 'node:fs'
+import { createLogger } from '@sim/logger'
+import { spawn } from 'bun'
+import { OPENAPI_SPEC_FILES } from '@/lib/openapi-specs'
+
+const logger = createLogger('DocsLlmsFullWatcher')
+const WATCHED_ROOT_FILES = new Set(['source.config.ts', ...OPENAPI_SPEC_FILES])
+
+let debounceTimer: ReturnType<typeof setTimeout> | undefined
+let pending = false
+let running = false
+
+async function regenerate(): Promise<void> {
+  running = true
+  try {
+    while (pending) {
+      pending = false
+      const child = spawn(['bun', 'run', 'llms-full:generate'], {
+        stdin: 'ignore',
+        stdout: 'inherit',
+        stderr: 'inherit',
+      })
+      const exitCode = await child.exited
+      if (exitCode !== 0) {
+        logger.warn('Documentation asset generation failed; retaining the previous asset', {
+          exitCode,
+        })
+      }
+    }
+  } finally {
+    running = false
+  }
+}
+
+function scheduleGeneration(): void {
+  pending = true
+  if (running) return
+  if (debounceTimer) clearTimeout(debounceTimer)
+  debounceTimer = setTimeout(() => {
+    debounceTimer = undefined
+    void regenerate()
+  }, 250)
+}
+
+for (const directory of ['content', 'lib']) {
+  watch(directory, { recursive: true }, scheduleGeneration)
+}
+
+watch('.', (_, changedPath) => {
+  if (changedPath && WATCHED_ROOT_FILES.has(changedPath.toString())) scheduleGeneration()
+})

apps/docs/scripts/watch-llms-full.ts

@@ -0,0 +1,8 @@
+{
+  "extends": ["//"],
+  "tasks": {
+    "build": {
+      "outputs": ["$TURBO_EXTENDS$", "public/llms-full.txt"]
+    }
+  }
+}