From 89892c53dd900e2d39ff781197a7964bd30814d3 Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Mon, 17 Aug 2026 16:54:57 +1200 Subject: [PATCH 01/11] Spike: Orama Builds an Orama index from the markdown llm-md-emitter already writes into dist/docs, and puts it behind the SearchEngine seam. Body text is indexed for the first time, so a phrase that appears in an article but not its title or headings is now findable. Reusing the emitted markdown means there is no second extraction pipeline: the emitter runs the same eligibility predicate search uses, so redirect stubs and navSearch:false pages are already gone. 1,253 pages indexed. The index is restored in a Web Worker. Restore is CPU-bound and would otherwise freeze the page for as long as it takes, with the overlay open and taking keystrokes. Uses Orama core save/load rather than @orama/plugin-data-persistence, which reaches for Node's filesystem and buffers and fails to bundle for a browser worker. Removes the search.json endpoint and the client scoring it fed. Co-Authored-By: Claude Opus 5 (1M context) --- astro.config.mjs | 4 + package.json | 3 +- pnpm-lock.yaml | 75 +------- src/components/DocsSearch.astro | 2 +- src/integrations/orama-index.ts | 134 ++++++++++++++ src/pages/docs/search.json.ts | 93 ---------- src/scripts/docs-search.ts | 4 +- src/scripts/modules/stemmer.js | 197 --------------------- src/scripts/modules/string.js | 83 --------- src/scripts/orama-worker.ts | 84 +++++++++ src/scripts/search-engine-legacy.ts | 265 ---------------------------- src/scripts/search-engine-orama.ts | 132 ++++++++++++++ src/scripts/synonyms.js | 17 -- 13 files changed, 367 insertions(+), 726 deletions(-) create mode 100644 src/integrations/orama-index.ts delete mode 100644 src/pages/docs/search.json.ts delete mode 100644 src/scripts/modules/stemmer.js delete mode 100644 src/scripts/modules/string.js create mode 100644 src/scripts/orama-worker.ts delete mode 100644 src/scripts/search-engine-legacy.ts create mode 100644 src/scripts/search-engine-orama.ts delete mode 100644 src/scripts/synonyms.js diff --git a/astro.config.mjs b/astro.config.mjs index a5f2c67a89..a96b699e39 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -3,6 +3,7 @@ import { satteri } from '@astrojs/markdown-satteri'; import mdx from '@astrojs/mdx'; import { attributeMarkdown, wrapTables } from '/src/themes/octopus/utilities/custom-markdown.mjs'; import llmMdEmitter from './src/integrations/llm-md-emitter.ts'; +import oramaIndex from './src/integrations/orama-index.ts'; import pruneDist from './src/integrations/prune-dist.ts'; import satteriHeadingId from './src/plugins/satteri-heading-id.js'; import satteriApiExamples, { apiExampleDirective } from './src/plugins/satteri-api-examples.js'; @@ -22,6 +23,9 @@ export default defineConfig({ integrations: [ mdx(), llmMdEmitter(), + // Indexes the markdown the emitter above just wrote, so it has to come + // after it and before the prune that would delete its output + oramaIndex(), // Must run last: strips build output that can't be served under /docs/ pruneDist() ], diff --git a/package.json b/package.json index 8ea81c1723..3c07ba4df1 100644 --- a/package.json +++ b/package.json @@ -33,13 +33,12 @@ "astro-accelerator-utils": "^0.3.84", "glob": "^13.0.6", "gray-matter": "^4.0.3", - "html-to-text": "^10.0.0", - "keyword-extractor": "^0.0.28", "optional": "^0.1.4", "satteri": "^0.9.5", "sharp": "^0.34.5" }, "devDependencies": { + "@orama/orama": "^3.1.18", "@playwright/test": "^1.60.0", "@pnpm/exe": "^11.0.9", "cspell": "^10.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 715655b317..a687f9e09e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -39,12 +39,6 @@ importers: gray-matter: specifier: ^4.0.3 version: 4.0.3 - html-to-text: - specifier: ^10.0.0 - version: 10.0.0 - keyword-extractor: - specifier: ^0.0.28 - version: 0.0.28 optional: specifier: ^0.1.4 version: 0.1.4 @@ -55,6 +49,9 @@ importers: specifier: ^0.34.5 version: 0.34.5 devDependencies: + '@orama/orama': + specifier: ^3.1.18 + version: 3.1.18 '@playwright/test': specifier: ^1.60.0 version: 1.62.0 @@ -815,6 +812,10 @@ packages: '@octopusdeploy/design-system-tokens@2026.3.8240': resolution: {integrity: sha512-SxabL5Y2bDD0kEEEtVUtTos3tiJ+2FEWTno1a40YVw/Y5eUbRnXUTChf4oVsRzw32GXpjBhxoTxiOBdOceeelA==} + '@orama/orama@3.1.18': + resolution: {integrity: sha512-a61ljmRVVyG5MC/698C8/FfFDw5a8LOIvyOLW5fztgUXqUpc1jOfQzOitSCbge657OgXXThmY3Tk8fpiDb4UcA==} + engines: {node: '>= 20.0.0'} + '@oslojs/encoding@1.1.0': resolution: {integrity: sha512-70wQhgYmndg4GCPxPPxPGevRKqTIJ2Nh4OkiMWmDAVYsTQ+Ta7Sq+rPevXyXGdzr30/qZBnyOalCszoMxlyldQ==} @@ -1008,11 +1009,6 @@ packages: '@rolldown/pluginutils@1.0.1': resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} - '@selderee/plugin-htmlparser2@0.12.0': - resolution: {integrity: sha512-oELmoyA6ML9jDRMV3kgcMQFKxUfBU0yFVn6yTctVaLT5ygXnxH52I3TZEgV9EhXJC68/uFvE5Daj1/25c0Xa/A==} - peerDependencies: - selderee: ~0.12.0 - '@shikijs/core@4.4.3': resolution: {integrity: sha512-QCR4q2ZO/ILJEuwiBMel4wdcTDb1JGwfjKTxPDF6x8ixOaluPrVqIn06C99AcRPhmYlBR56d/Fb+GN58GzExpg==} engines: {node: '>=20'} @@ -1398,10 +1394,6 @@ packages: decode-named-character-reference@1.3.0: resolution: {integrity: sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==} - deepmerge-ts@7.1.5: - resolution: {integrity: sha512-HOJkrhaYsweh+W+e74Yn7YStZOilkoPb6fycpwNLKzSPtruFs48nYis0zy5yJz1+ktUhHxoRDJ27RQAWLIJVJw==} - engines: {node: '>=16.0.0'} - define-data-property@1.1.4: resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} engines: {node: '>= 0.4'} @@ -1778,10 +1770,6 @@ packages: html-escaper@3.0.3: resolution: {integrity: sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==} - html-to-text@10.0.0: - resolution: {integrity: sha512-2OH59Gtprdczel+7Rxgpz9hGVJREaf8Lt1H4kZwWHpEn70VQKRuMNGsb2eDbwaTzrYzb0hheiOG1P7Dim0B4dQ==} - engines: {node: '>=20.19.0'} - html-void-elements@3.0.0: resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} @@ -1979,17 +1967,10 @@ packages: jsonc-parser@3.3.1: resolution: {integrity: sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==} - keyword-extractor@0.0.28: - resolution: {integrity: sha512-oi7dSPpYtW/3fE0vZiqQgZ8mW3F1V9K4+rBJ0FcVrdXBEQuhZ0zKj7sX74eqGASuepLHf9aYdeonyKHWhYpHQA==} - engines: {node: '>= 0.10.0'} - kind-of@6.0.3: resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} engines: {node: '>=0.10.0'} - leac@0.7.0: - resolution: {integrity: sha512-qMrZeyEekgdRQ9o6a4NAB2EQZrv827GJdn1vnapwSJ90hWRB4TzUSunvacPkxQ2TnNqHNI1/zSt0hlo0crG8Jw==} - lightningcss-android-arm64@1.33.0: resolution: {integrity: sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==} engines: {node: '>= 12.0.0'} @@ -2405,9 +2386,6 @@ packages: parse5@7.3.0: resolution: {integrity: sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==} - parseley@0.13.1: - resolution: {integrity: sha512-uNBJZzmb60l6p6VWLTmevizNAGnE0xoSf1n0B4q3ntegDNzcS68NRCcBDZTcyXHxt2XhBChsCuqj4M+nChvE/A==} - path-key@2.0.1: resolution: {integrity: sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==} engines: {node: '>=4'} @@ -2431,9 +2409,6 @@ packages: resolution: {integrity: sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==} engines: {node: '>=4'} - peberminta@0.10.0: - resolution: {integrity: sha512-80B2AsU+I4Qdb0ZAPSfe9UwvGzwkM37IKIFEvdS3D/3Ndgv2bsuJ0bfG1+iEYO+l7Gfd4EUJmuRyq7efLgRMzQ==} - piccolore@0.1.3: resolution: {integrity: sha512-o8bTeDWjE086iwKrROaDf31K0qC/BENdm15/uH9usSC/uZjJOKb2YGiVHfLY4GhwsERiPI1jmwI2XrA7ACOxVw==} @@ -2634,9 +2609,6 @@ packages: resolution: {integrity: sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==} engines: {node: '>=4'} - selderee@0.12.0: - resolution: {integrity: sha512-b1YMh3+DHZp59DLna3qVwQ5iOla/nrI6mLBNW02XxU77M3046Df6VLkoaJyFz20VsGIG5kkp+FK0kg4K4HnUFw==} - semver@5.7.2: resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} hasBin: true @@ -3792,6 +3764,8 @@ snapshots: '@octopusdeploy/design-system-tokens@2026.3.8240': {} + '@orama/orama@3.1.18': {} + '@oslojs/encoding@1.1.0': {} '@oxc-project/types@0.144.0': {} @@ -3916,12 +3890,6 @@ snapshots: '@rolldown/pluginutils@1.0.1': {} - '@selderee/plugin-htmlparser2@0.12.0(selderee@0.12.0)': - dependencies: - domelementtype: 2.3.0 - domhandler: 5.0.3 - selderee: 0.12.0 - '@shikijs/core@4.4.3': dependencies: '@shikijs/primitive': 4.4.3 @@ -4437,8 +4405,6 @@ snapshots: dependencies: character-entities: 2.0.2 - deepmerge-ts@7.1.5: {} - define-data-property@1.1.4: dependencies: es-define-property: 1.0.1 @@ -4998,14 +4964,6 @@ snapshots: html-escaper@3.0.3: {} - html-to-text@10.0.0: - dependencies: - '@selderee/plugin-htmlparser2': 0.12.0(selderee@0.12.0) - deepmerge-ts: 7.1.5 - dom-serializer: 2.0.0 - htmlparser2: 10.1.0 - selderee: 0.12.0 - html-void-elements@3.0.0: {} htmlparser2@10.1.0: @@ -5193,12 +5151,8 @@ snapshots: jsonc-parser@3.3.1: {} - keyword-extractor@0.0.28: {} - kind-of@6.0.3: {} - leac@0.7.0: {} - lightningcss-android-arm64@1.33.0: optional: true @@ -5879,11 +5833,6 @@ snapshots: dependencies: entities: 6.0.1 - parseley@0.13.1: - dependencies: - leac: 0.7.0 - peberminta: 0.10.0 - path-key@2.0.1: {} path-key@3.1.1: {} @@ -5904,8 +5853,6 @@ snapshots: dependencies: pify: 3.0.0 - peberminta@0.10.0: {} - piccolore@0.1.3: {} picocolors@1.1.1: {} @@ -6198,10 +6145,6 @@ snapshots: extend-shallow: 2.0.1 kind-of: 6.0.3 - selderee@0.12.0: - dependencies: - parseley: 0.13.1 - semver@5.7.2: {} semver@7.8.0: {} diff --git a/src/components/DocsSearch.astro b/src/components/DocsSearch.astro index 4957e1b73c..c09bbaf793 100644 --- a/src/components/DocsSearch.astro +++ b/src/components/DocsSearch.astro @@ -44,7 +44,7 @@ if (facets.length === 0) { throw new Error('DocsSearch needs at least one facet for its tab strip'); } -const indexUrl = `${SITE.subfolder}/search.json`; +const indexUrl = `${SITE.subfolder}/search-index.json`; const listboxId = `${name}-search-listbox`; --- diff --git a/src/integrations/orama-index.ts b/src/integrations/orama-index.ts new file mode 100644 index 0000000000..c54c1bb7d5 --- /dev/null +++ b/src/integrations/orama-index.ts @@ -0,0 +1,134 @@ +// Builds an Orama index and serialises it for the browser to restore. +// +// The corpus is the `.md` files `llm-md-emitter.ts` has already written into +// `dist/docs/`. That emitter runs the same eligibility predicate search uses, so +// redirect stubs and `navSearch: false` pages are already gone and there is no +// second extraction pipeline to keep in step with the first. This has to be +// registered after `llmMdEmitter()` and before `pruneDist()`, which would delete +// anything written outside `dist/docs/`. + +import type { AstroIntegration } from 'astro'; +import { fileURLToPath } from 'node:url'; +import * as path from 'node:path'; +import * as fs from 'node:fs'; +import { globSync } from 'glob'; +// Core `save` rather than @orama/plugin-data-persistence: the plugin reaches for +// Node's filesystem and buffers, and bundling it for the browser worker that has +// to restore this fails outright. `save` returns a plain object, which is all +// the JSON format ever was. +import { create, insertMultiple, save } from '@orama/orama'; + +// Roughly the number of pages that pass the search predicate. Well under it and +// the corpus has moved, which is a silent failure otherwise. +const EXPECTED_PAGES = 1000; + +// Long pages cost index size for very little relevance — the terms that matter +// are near the top, and a result excerpt never reaches further than this. +const BODY_LIMIT = 4000; + +const SECTIONS: [string, RegExp][] = [ + [ + 'cli', + /^\/docs\/octopus-rest-api\/(cli|octopus-cli|[a-z.]+-command-line)(\/|$)/, + ], + ['api', /^\/docs\/octopus-rest-api(\/|$)/], + ['integrations', /^\/docs\/api-and-integration(\/|$)/], +]; + +function sectionFor(url: string) { + return SECTIONS.find(([, prefix]) => prefix.test(url))?.[0] ?? 'docs'; +} + +function trailFor(url: string) { + return url + .split('/') + .filter(Boolean) + .slice(1, -1) + .map((segment) => + segment.replace(/-/g, ' ').replace(/^./, (c) => c.toUpperCase()) + ) + .join(' / '); +} + +/** Strips the markup that would otherwise be indexed as words in its own right: + * fences, link syntax, heading markers and emphasis. */ +function toPlainText(markdown: string) { + return markdown + .replace(/```[\s\S]*?```/g, ' ') + .replace(/`([^`]*)`/g, '$1') + .replace(/!\[[^\]]*\]\([^)]*\)/g, ' ') + .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1') + .replace(/^[#>\s-]+/gm, ' ') + .replace(/[*_]{1,3}/g, '') + .replace(/\s+/g, ' ') + .trim(); +} + +export default function oramaIndex(): AstroIntegration { + return { + name: 'orama-index', + hooks: { + 'astro:build:done': async ({ dir, logger }) => { + const distDir = fileURLToPath(dir); + const docsDir = path.join(distDir, 'docs'); + + const files = globSync('**/*.md', { + cwd: docsDir, + nodir: true, + posix: true, + }); + + const documents = files.map((file) => { + const raw = fs + .readFileSync(path.join(docsDir, file), 'utf8') + .replace(/^/, ''); + + // The emitter writes `# Title`, an optional `> subtitle`, then body. + const [, title = ''] = raw.match(/^#\s+(.*)$/m) ?? []; + const [, subtitle = ''] = raw.match(/^>\s+(.*)$/m) ?? []; + const url = '/docs/' + file.replace(/\.md$/, ''); + + return { + url, + title, + description: subtitle, + body: toPlainText(raw).slice(0, BODY_LIMIT), + trail: trailFor(url), + section: sectionFor(url), + }; + }); + + const db = create({ + schema: { + url: 'string', + title: 'string', + description: 'string', + body: 'string', + trail: 'string', + section: 'enum', + } as const, + components: { + tokenizer: { stemming: true, language: 'english' }, + }, + }); + + await insertMultiple(db, documents); + + const serialised = JSON.stringify(save(db)); + const target = path.join(docsDir, 'search-index.json'); + fs.writeFileSync(target, serialised, 'utf8'); + + const megabytes = (serialised.length / 1024 / 1024).toFixed(1); + logger.info( + `indexed ${documents.length} pages into docs/search-index.json (${megabytes}MB)` + ); + + if (documents.length < EXPECTED_PAGES) { + logger.warn( + `only ${documents.length} pages were indexed, expected at least ${EXPECTED_PAGES} — check llm-md-emitter still runs before this` + ); + } + }, + }, + }; +} diff --git a/src/pages/docs/search.json.ts b/src/pages/docs/search.json.ts deleted file mode 100644 index 6ef6c53a15..0000000000 --- a/src/pages/docs/search.json.ts +++ /dev/null @@ -1,93 +0,0 @@ -/** @format */ - -// warning: This file is overwritten by Astro Accelerator - -import { accelerator } from '@lib/accelerator'; -import { PostFiltering } from 'astro-accelerator-utils'; -import type { MarkdownInstance } from 'astro'; -import { SITE } from '@config'; -import { convert } from 'html-to-text'; -import keywordExtractor from 'keyword-extractor'; -import { isUnderConstruction } from '@lib/underConstruction'; - -const getData = async () => { - //@ts-ignore - const allPages = import.meta.glob(['./**/*.md', './**/*.mdx']); - const items = []; - - for (const path in allPages) { - // Temporary - see src/lib/underConstruction.ts. - if (isUnderConstruction(path)) { - continue; - } - - const page = (await allPages[path]()) as MarkdownInstance< - Record - >; - - if (!PostFiltering.showInSearch(page)) { - continue; - } - - let url = page.url ?? ''; - - if (page.frontmatter.paged) { - url += '/1/'; - } - - const headings = await page.getHeadings(); - const title = await accelerator.markdown.getTextFrom( - page.frontmatter?.title - ); - const content = page.compiledContent ? await page.compiledContent() : ''; - let counted: { word: string; count: number }[] = []; - - if (content) { - const options = { - wordwrap: false, - selectors: [ - { selector: 'a', options: { ignoreHref: true } }, - { selector: 'img', format: 'skip' }, - { selector: 'h1', options: { uppercase: false } }, - ], - }; - const text = convert(content, options); - - const words = keywordExtractor.extract(text, { - language: 'english', - return_changed_case: true, - remove_duplicates: true, - }); - - counted = words - .map((w) => { - return { - word: w, - count: words.filter((wd) => wd === w).length, - }; - }) - .filter((e) => e.word.replace(/[^a-z]+/g, '').length > 1); - } - - items.push({ - title: title, - headings: headings.map((h) => { - return { text: h.text, slug: h.slug }; - }), - description: page.frontmatter.description ?? '', - keywords: counted.map((c) => c.word).join(' '), - tags: page.frontmatter.tags ?? [], - url: SITE.url + accelerator.urlFormatter.formatAddress(url), - date: page.frontmatter.pubDate ?? '', - }); - } - - return new Response(JSON.stringify(items), { - status: 200, - headers: { - 'Content-Type': 'application/json', - }, - }); -}; - -export const GET = getData; diff --git a/src/scripts/docs-search.ts b/src/scripts/docs-search.ts index ef88eb5758..c22dc25e14 100644 --- a/src/scripts/docs-search.ts +++ b/src/scripts/docs-search.ts @@ -14,7 +14,7 @@ import { type SearchEngine, type SearchResult, } from './search-engine'; -import { legacyEngine } from './search-engine-legacy'; +import { oramaEngine } from './search-engine-orama'; const DEBOUNCE_MS = 150; const SITE_SEARCH = 'site'; @@ -41,7 +41,7 @@ function setup(dialog: HTMLDialogElement) { const demo = dialog.dataset.demoResults; const engine: SearchEngine = demo ? fixtureEngine(JSON.parse(demo)) - : legacyEngine(dialog.dataset.indexUrl ?? '/docs/search.json'); + : oramaEngine(dialog.dataset.indexUrl ?? '/docs/search-index.json'); let facet = 'all'; let active = -1; diff --git a/src/scripts/modules/stemmer.js b/src/scripts/modules/stemmer.js deleted file mode 100644 index 778ebcc9c0..0000000000 --- a/src/scripts/modules/stemmer.js +++ /dev/null @@ -1,197 +0,0 @@ -/** - * Removes "morphological and inflexional endings" from words - * See: http://www.tartarus.org/~martin/PorterStemmer - */ -export const stemmer = (function () { - const step2list = { - ational: 'ate', - tional: 'tion', - enci: 'ence', - anci: 'ance', - izer: 'ize', - bli: 'ble', - alli: 'al', - entli: 'ent', - eli: 'e', - ousli: 'ous', - ization: 'ize', - ation: 'ate', - ator: 'ate', - alism: 'al', - iveness: 'ive', - fulness: 'ful', - ousness: 'ous', - aliti: 'al', - iviti: 'ive', - biliti: 'ble', - logi: 'log', - }; - - const step3list = { - icate: 'ic', - ative: '', - alize: 'al', - iciti: 'ic', - ical: 'ic', - ful: '', - ness: '', - }; - - const c = '[^aeiou]', // consonant - v = '[aeiouy]', // vowel - C = c + '[^aeiouy]*', // consonant sequence - V = v + '[aeiou]*', // vowel sequence - mgr0 = '^(' + C + ')?' + V + C, // [C]VC... is m>0 - meq1 = '^(' + C + ')?' + V + C + '(' + V + ')?$', // [C]VC[V] is m=1 - mgr1 = '^(' + C + ')?' + V + C + V + C, // [C]VCVC... is m>1 - s_v = '^(' + C + ')?' + v; // vowel in stem - - /** - * @param {string} w - * @returns {string} - */ - return function (w) { - var stem, - suffix, - firstch, - re, - re2, - re3, - re4, - origword = w; - - if (w.length < 3) { - return w; - } - - firstch = w.substring(0, 1); - - if (firstch == 'y') { - w = firstch.toUpperCase() + w.substring(1, w.length); - } - - // Step 1a - re = /^(.+?)(ss|i)es$/; - re2 = /^(.+?)([^s])s$/; - - if (re.test(w)) { - w = w.replace(re, '$1$2'); - } else if (re2.test(w)) { - w = w.replace(re2, '$1$2'); - } - - // Step 1b - re = /^(.+?)eed$/; - re2 = /^(.+?)(ed|ing)$/; - if (re.test(w)) { - var fp = re.exec(w); - re = new RegExp(mgr0); - if (re.test(fp[1])) { - re = /.$/; - w = w.replace(re, ''); - } - } else if (re2.test(w)) { - var fp = re2.exec(w); - stem = fp[1]; - re2 = new RegExp(s_v); - if (re2.test(stem)) { - w = stem; - re2 = /(at|bl|iz)$/; - re3 = new RegExp('([^aeiouylsz])\\1$'); - re4 = new RegExp('^' + C + v + '[^aeiouwxy]$'); - if (re2.test(w)) { - w = w + 'e'; - } else if (re3.test(w)) { - re = /.$/; - w = w.replace(re, ''); - } else if (re4.test(w)) { - w = w + 'e'; - } - } - } - - // Step 1c - re = /^(.+?)y$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - re = new RegExp(s_v); - if (re.test(stem)) { - w = stem + 'i'; - } - } - - // Step 2 - re = - /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - suffix = fp[2]; - re = new RegExp(mgr0); - if (re.test(stem)) { - w = stem + step2list[suffix]; - } - } - - // Step 3 - re = /^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - suffix = fp[2]; - re = new RegExp(mgr0); - if (re.test(stem)) { - w = stem + step3list[suffix]; - } - } - - // Step 4 - re = - /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/; - re2 = /^(.+?)(s|t)(ion)$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - re = new RegExp(mgr1); - if (re.test(stem)) { - w = stem; - } - } else if (re2.test(w)) { - var fp = re2.exec(w); - stem = fp[1] + fp[2]; - re2 = new RegExp(mgr1); - if (re2.test(stem)) { - w = stem; - } - } - - // Step 5 - re = /^(.+?)e$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - re = new RegExp(mgr1); - re2 = new RegExp(meq1); - re3 = new RegExp('^' + C + v + '[^aeiouwxy]$'); - if (re.test(stem) || (re2.test(stem) && !re3.test(stem))) { - w = stem; - } - } - - re = /ll$/; - re2 = new RegExp(mgr1); - if (re.test(w) && re2.test(w)) { - re = /.$/; - w = w.replace(re, ''); - } - - // and turn initial Y back to y - - if (firstch == 'y') { - w = firstch.toLowerCase() + w.substr(1); - } - - return w; - }; -})(); diff --git a/src/scripts/modules/string.js b/src/scripts/modules/string.js deleted file mode 100644 index dea7142565..0000000000 --- a/src/scripts/modules/string.js +++ /dev/null @@ -1,83 +0,0 @@ -// @ts-check - -/** - * Looks for a search within a string - * - * @param {string} string - * @param {string} search - * @returns - */ -function contains(string, search) { - return string.indexOf(search) > -1; -} - -/** - * Looks for a search within a string - * - * @param {string} string - * @param {string} search - * @returns - */ -function containsWord(string, search) { - return string.split(' ').indexOf(search) > -1; -} - -/** - * - * @param {string} string - * @param {string[]} terms - * @returns - */ -function highlight(string, terms) { - terms.forEach((term) => { - const regEx = new RegExp(term, 'ig'); - const matches = string.match(regEx); - if (matches) { - string = string.replace(regEx, `${matches[0]}`); - } - }); - return string; -} - -/** - * Simplifies a string to plain lower case, removing diacritic characters and hyphens - * This means a search for "co-op" will be found in "COOP" and "Café" will be found in "cafe" - * @param {string} string - * @returns {string} - */ -function sanitise(string) { - // @ts-ignore - if (String.prototype.normalize) { - // Reduces diacritic characters to plain characters - string - .trim() - .normalize('NFD') - .replace(/\./g, ' ') - .replace(/[\u0300-\u036f]/g, '') - .toLowerCase() - .replace(/-/g, ''); - } - - // Some browsers can't normalise strings - return string.trim().toLowerCase().replace(/-/g, ''); -} - -/** - * Sets a minimum length for a search - * @param {string} string - * @returns - */ -function isLongEnough(string) { - return string.length > 1; -} - -/** - * - * @param {string} string - * @returns {string[]} - */ -function explode(string) { - return string.split(' ').filter(isLongEnough).map(sanitise); -} - -export { contains, containsWord, sanitise, explode, highlight }; diff --git a/src/scripts/orama-worker.ts b/src/scripts/orama-worker.ts new file mode 100644 index 0000000000..f51bef3bf4 --- /dev/null +++ b/src/scripts/orama-worker.ts @@ -0,0 +1,84 @@ +// Holds the Orama index off the main thread. +// +// Restoring a serialised index is CPU-bound and proportional to the corpus, so +// on the main thread it would freeze the page for as long as it takes. The +// overlay is open and taking keystrokes while this runs. +// +// Core `create` + `load` rather than @orama/plugin-data-persistence: the plugin +// reaches for Node's filesystem and buffers, and bundling it for a browser +// worker fails at import time. + +import { create, load, search } from '@orama/orama'; + +type LoadMessage = { type: 'load'; indexUrl: string }; +type SearchMessage = { + type: 'search'; + id: number; + query: string; + limit: number; +}; + +// Has to match the schema the index was built with, or `load` restores an index +// the search side cannot read. +const SCHEMA = { + url: 'string', + title: 'string', + description: 'string', + body: 'string', + trail: 'string', + section: 'enum', +} as const; + +let indexUrl = ''; +let ready: Promise | null = null; + +function open() { + ready ??= fetch(indexUrl) + .then((response) => response.json()) + .then((raw) => { + const db = create({ schema: SCHEMA }); + load(db, raw); + return db; + }) + .catch((error) => { + // A failed load must not poison every later search. + ready = null; + throw error; + }); + + return ready; +} + +self.addEventListener( + 'message', + async (event: MessageEvent) => { + const message = event.data; + + if (message.type === 'load') { + indexUrl = message.indexUrl; + // Kicked off here so the download and restore overlap with typing; the + // rejection is handled inside `open`. + open().catch(() => {}); + return; + } + + try { + const db = await open(); + const results = await search(db as never, { + term: message.query, + limit: message.limit, + // Body carries the most text and the least signal per word, so it is + // weighted below the fields that name what a page is about. + boost: { title: 4, description: 2, trail: 2, body: 1 }, + tolerance: 1, + }); + + self.postMessage({ + id: message.id, + hits: results.hits.map((hit) => hit.document), + }); + } catch (error) { + self.postMessage({ id: message.id, error: String(error) }); + } + } +); diff --git a/src/scripts/search-engine-legacy.ts b/src/scripts/search-engine-legacy.ts deleted file mode 100644 index 398c758757..0000000000 --- a/src/scripts/search-engine-legacy.ts +++ /dev/null @@ -1,265 +0,0 @@ -// The current search, behind the `SearchEngine` seam: one build-time JSON of -// titles, headings, descriptions, tags and an extracted keyword bag, scored in -// the browser. The scoring is carried over unchanged from the results list this -// replaces, so the overlay returns exactly what the old dropdown returned. -// -// Body text is not in the index, which is the ceiling on how good this can be -// and the reason the Pagefind and Orama spikes exist. - -import { contains, explode, highlight, sanitise } from './modules/string.js'; -import { stemmer } from './modules/stemmer.js'; -import { - breadcrumbFrom, - classify, - countByFacet, - type SearchEngine, - type SearchResult, -} from './search-engine'; - -type Heading = { text: string; slug: string; safeText: string }; - -type Entry = { - title: string; - safeTitle: string; - description: string; - keywords: string; - tags: string[]; - headings: Heading[]; - url: string; - depth: number; -}; - -type Scored = Entry & { - score: number; - foundWords: number; - foundTerms: string[]; -}; - -// Phrase and per-term weights, verbatim from the list this replaces. -const SCORING = { - depth: 5, - phraseTitle: 60, - phraseHeading: 20, - phraseDescription: 20, - termTitle: 40, - termHeading: 15, - termDescription: 15, - termTags: 15, - termKeywords: 15, -}; - -const WORD_SCORES = { - titleExact: 20, - titleContains: 15, - headingContains: 10, - contentContains: 1, -}; - -const RESULT_LIMIT = 30; - -async function getSynonyms(): Promise> { - try { - return (await import('./synonyms.js')).synonyms; - } catch { - return {}; - } -} - -/** Terms reach `highlight()` as a regular expression, so a query like `c++` - * would throw before anything was drawn. */ -function escapeForRegExp(term: string) { - return term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); -} - -/** The excerpt is set as HTML so the `` around each hit survives. Escaping - * first means those marks are the only tags that can ever reach the DOM. */ -function escapeHtml(text: string) { - return text - .replace(/&/g, '&') - .replace(//g, '>'); -} - -async function expand(query: string) { - const synonyms = await getSynonyms(); - const terms: string[] = []; - - for (const term of explode(query)) { - const synonym = synonyms[term]; - // An empty synonym is a stop word: drop the term rather than expanding it. - if (synonym === '') continue; - terms.push(term); - if (synonym) terms.push(...synonym.split(' ')); - } - - const stemmed = terms - .map((term) => stemmer(term)) - .filter((stem) => !terms.includes(stem)); - - return { terms, allTerms: [...terms, ...stemmed] }; -} - -function score(entry: Entry, query: string, allTerms: string[]): Scored { - const scored: Scored = { ...entry, score: 0, foundWords: 0, foundTerms: [] }; - - // Phrase matches: the whole query, found intact - if (scored.safeTitle === query) scored.foundWords += WORD_SCORES.titleExact; - - if (contains(scored.safeTitle, query)) { - scored.score += SCORING.phraseTitle; - scored.foundWords += WORD_SCORES.titleContains; - } - - for (const heading of scored.headings) { - if (contains(heading.safeText, query)) { - scored.score += SCORING.phraseHeading; - scored.foundWords += WORD_SCORES.headingContains; - } - } - - if (contains(scored.description, query)) { - scored.score += SCORING.phraseDescription; - scored.foundWords += WORD_SCORES.contentContains; - } - - // Term matches: each word of the query, found anywhere - for (const term of allTerms) { - let found = false; - - if (contains(scored.safeTitle, term)) { - scored.score += SCORING.termTitle; - scored.foundWords += WORD_SCORES.headingContains / 2; - found = true; - } - - for (const heading of scored.headings) { - if (contains(heading.safeText, term)) { - scored.score += SCORING.termHeading; - found = true; - } - } - - if (contains(scored.description, term)) { - scored.score += SCORING.termDescription; - found = true; - } - - for (const tag of scored.tags) { - if (contains(tag, term)) { - scored.score += SCORING.termTags; - found = true; - } - } - - if (contains(scored.keywords, term)) { - scored.score += SCORING.termKeywords; - found = true; - } - - if (found) { - scored.foundWords++; - if (!scored.foundTerms.includes(term)) scored.foundTerms.push(term); - } - } - - // Shallow pages win ties: /docs/features over /docs/features/a/b - if (scored.score > 0) { - if (scored.depth < 5) { - scored.score += SCORING.depth; - scored.foundWords++; - } - if (scored.depth < 4) { - scored.score += SCORING.depth; - scored.foundWords++; - } - } - - return scored; -} - -function byRelevance(a: Scored, b: Scored) { - if (b.foundTerms.length !== a.foundTerms.length) { - return b.foundTerms.length - a.foundTerms.length; - } - if (b.foundWords !== a.foundWords) return b.foundWords - a.foundWords; - return b.score - a.score; -} - -export function legacyEngine(indexUrl: string): SearchEngine { - // One fetch, on the first search rather than on page load, shared by every - // search after it. The old list fetched 1.86MB the moment the script ran. - let loading: Promise | null = null; - - function load() { - loading ??= fetch(indexUrl) - .then((response) => response.json()) - .then((data: Record[]) => - data.map((item) => ({ - title: item.title, - safeTitle: sanitise(item.title), - description: item.description ?? '', - keywords: item.keywords ?? '', - tags: (item.tags ?? []).map((tag: string) => sanitise(tag)), - headings: (item.headings ?? []).map((heading: Heading) => ({ - ...heading, - safeText: sanitise(heading.text), - })), - url: item.url, - depth: item.url.match(/\//g)?.length ?? 0, - })) - ) - .catch(() => { - // A failed load must not poison every later search. - loading = null; - return [] as Entry[]; - }); - - return loading; - } - - return { - warm: load, - - async search(rawQuery, facet) { - // Chained words are joined, so `System.Text` searches as `systemtext`. - const query = sanitise(rawQuery.replace(/\./g, ' ')); - if (!query) return { results: [], counts: countByFacet([]) }; - - const [haystack, { terms, allTerms }] = await Promise.all([ - load(), - expand(query), - ]); - - const highlightTerms = terms.map(escapeForRegExp); - - const matched = haystack - .map((entry) => score(entry, query, allTerms)) - .filter((entry) => entry.score > 0) - .sort(byRelevance) - .slice(0, RESULT_LIMIT) - .map((entry): SearchResult => { - // The index stores absolute production URLs, and every entry in it is - // a page of this site. Keeping only the path is what makes a result - // stay on whatever host is serving it — an ephemeral environment or - // localhost would otherwise send you to production. - const path = new URL(entry.url, window.location.origin).pathname; - - return { - url: path, - title: entry.title, - excerpt: highlight(escapeHtml(entry.description), highlightTerms), - breadcrumb: breadcrumbFrom(path), - ...classify(path), - }; - }); - - return { - counts: countByFacet(matched), - results: - facet && facet !== 'all' - ? matched.filter((result) => result.facet === facet) - : matched, - }; - }, - }; -} diff --git a/src/scripts/search-engine-orama.ts b/src/scripts/search-engine-orama.ts new file mode 100644 index 0000000000..cc2a923908 --- /dev/null +++ b/src/scripts/search-engine-orama.ts @@ -0,0 +1,132 @@ +// Orama behind the `SearchEngine` seam. +// +// Unlike Pagefind, the whole index is downloaded and restored into memory before +// the first query can run — there is no per-query fetch at all after that. That +// is the trade the spike exists to measure: a larger up-front cost against zero +// marginal cost, on a corpus this size. +// +// Restore is CPU-bound and would block the main thread, so it happens in a +// worker and queries are messaged across. + +import { + classify, + countByFacet, + type SearchEngine, + type SearchResult, +} from './search-engine'; + +const RESULT_LIMIT = 30; + +type WorkerHit = { + url: string; + title: string; + description: string; + body: string; + trail: string; +}; + +type WorkerReply = { + id: number; + hits?: WorkerHit[]; + error?: string; +}; + +function escapeHtml(text: string) { + return text + .replace(/&/g, '&') + .replace(//g, '>'); +} + +function escapeForRegExp(term: string) { + return term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +/** + * A window of body text around the first term that matched, with the terms + * marked. Orama returns whole documents rather than excerpts, so the result row + * has to cut its own. + */ +function excerptFrom(hit: WorkerHit, terms: string[]) { + const source = hit.description || hit.body; + if (!source) return ''; + + const pattern = new RegExp(terms.map(escapeForRegExp).join('|'), 'i'); + const at = source.search(pattern); + const from = at > 60 ? source.lastIndexOf(' ', at - 60) + 1 : 0; + const window = source.slice(from, from + 180); + + const text = escapeHtml((from > 0 ? '…' : '') + window); + return text.replace( + new RegExp(terms.map(escapeForRegExp).join('|'), 'gi'), + (match) => `${match}` + ); +} + +export function oramaEngine(indexUrl: string): SearchEngine { + let worker: Worker | null = null; + let nextId = 0; + const pending = new Map void>(); + + function load() { + if (worker) return; + + worker = new Worker(new URL('./orama-worker.ts', import.meta.url), { + type: 'module', + }); + worker.addEventListener('message', (event: MessageEvent) => { + pending.get(event.data.id)?.(event.data); + pending.delete(event.data.id); + }); + worker.postMessage({ type: 'load', indexUrl }); + } + + function ask(query: string, facet?: string) { + load(); + const id = nextId++; + + return new Promise((resolve) => { + pending.set(id, resolve); + worker!.postMessage({ type: 'search', id, query, facet, limit: 200 }); + }); + } + + return { + warm: load, + + async search(rawQuery, facet) { + const query = rawQuery.trim(); + if (!query) return { results: [], counts: countByFacet([]) }; + + // Counts come from the whole match set rather than the filtered one, so + // the strip keeps showing what the other tabs hold while one is selected. + const reply = await ask(query); + if (reply.error || !reply.hits) { + return { results: [], counts: countByFacet([]) }; + } + + const terms = query.split(/\s+/).filter((term) => term.length > 1); + + const all = reply.hits.map((hit): SearchResult => { + const path = new URL(hit.url, window.location.origin).pathname; + return { + url: path, + title: hit.title || path, + excerpt: excerptFrom(hit, terms.length > 0 ? terms : [query]), + breadcrumb: hit.trail ? hit.trail.split(' / ') : [], + ...classify(path), + }; + }); + + const shown = + facet && facet !== 'all' + ? all.filter((result) => result.facet === facet) + : all; + + return { + results: shown.slice(0, RESULT_LIMIT), + counts: countByFacet(all), + }; + }, + }; +} diff --git a/src/scripts/synonyms.js b/src/scripts/synonyms.js deleted file mode 100644 index 489fc14133..0000000000 --- a/src/scripts/synonyms.js +++ /dev/null @@ -1,17 +0,0 @@ -const synonyms = { - // Keep me alphabetical - licence: 'license', - logs: 'log', - regex: 'regular expression', - // Join words - a: '', - and: '', - for: '', - if: '', - the: '', - to: '', - via: '', - with: '', -}; - -export { synonyms }; From 4b5ce5762009a3b92d4b1f059d4f51ab110c8e41 Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Tue, 18 Aug 2026 09:28:05 +1200 Subject: [PATCH 02/11] Add orama to the project dictionary, and use US spelling Co-Authored-By: Claude Opus 5 (1M context) --- dictionary-octopus.txt | 1 + src/integrations/orama-index.ts | 8 ++++---- src/scripts/orama-worker.ts | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/dictionary-octopus.txt b/dictionary-octopus.txt index cdcca35cbf..7c66dcf4f6 100644 --- a/dictionary-octopus.txt +++ b/dictionary-octopus.txt @@ -399,6 +399,7 @@ onlylogs onmicrosoft openon operationalize +orama outfile outputformat OWASP diff --git a/src/integrations/orama-index.ts b/src/integrations/orama-index.ts index c54c1bb7d5..f3d9a2b075 100644 --- a/src/integrations/orama-index.ts +++ b/src/integrations/orama-index.ts @@ -1,4 +1,4 @@ -// Builds an Orama index and serialises it for the browser to restore. +// Builds an Orama index and serializes it for the browser to restore. // // The corpus is the `.md` files `llm-md-emitter.ts` has already written into // `dist/docs/`. That emitter runs the same eligibility predicate search uses, so @@ -114,11 +114,11 @@ export default function oramaIndex(): AstroIntegration { await insertMultiple(db, documents); - const serialised = JSON.stringify(save(db)); + const serialized = JSON.stringify(save(db)); const target = path.join(docsDir, 'search-index.json'); - fs.writeFileSync(target, serialised, 'utf8'); + fs.writeFileSync(target, serialized, 'utf8'); - const megabytes = (serialised.length / 1024 / 1024).toFixed(1); + const megabytes = (serialized.length / 1024 / 1024).toFixed(1); logger.info( `indexed ${documents.length} pages into docs/search-index.json (${megabytes}MB)` ); diff --git a/src/scripts/orama-worker.ts b/src/scripts/orama-worker.ts index f51bef3bf4..fb99108dad 100644 --- a/src/scripts/orama-worker.ts +++ b/src/scripts/orama-worker.ts @@ -1,6 +1,6 @@ // Holds the Orama index off the main thread. // -// Restoring a serialised index is CPU-bound and proportional to the corpus, so +// Restoring a serialized index is CPU-bound and proportional to the corpus, so // on the main thread it would freeze the page for as long as it takes. The // overlay is open and taking keystrokes while this runs. // From 6fa97ae5c9c803035aa0a71c255abb0ec9c6b4f9 Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Tue, 18 Aug 2026 12:54:49 +1200 Subject: [PATCH 03/11] Restore the Orama index with the tokenizer it was built with The worker created its database with only a schema, so query terms were tokenized without the English stemmer the index was built with. Searching `variables` matched 5 pages instead of 321, and typo tolerance matched nothing at all. Search kept returning results throughout, so nothing failed loudly. Co-Authored-By: Claude Opus 5 (1M context) --- src/integrations/orama-index.ts | 3 +++ src/scripts/orama-worker.ts | 13 ++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/integrations/orama-index.ts b/src/integrations/orama-index.ts index f3d9a2b075..d53aefcc06 100644 --- a/src/integrations/orama-index.ts +++ b/src/integrations/orama-index.ts @@ -107,6 +107,9 @@ export default function oramaIndex(): AstroIntegration { trail: 'string', section: 'enum', } as const, + // `orama-worker.ts` has to create its database with this same + // tokenizer before it restores the index, or query terms are matched + // unstemmed against stemmed index terms and recall collapses quietly. components: { tokenizer: { stemming: true, language: 'english' }, }, diff --git a/src/scripts/orama-worker.ts b/src/scripts/orama-worker.ts index fb99108dad..e108ec7fe2 100644 --- a/src/scripts/orama-worker.ts +++ b/src/scripts/orama-worker.ts @@ -29,6 +29,14 @@ const SCHEMA = { section: 'enum', } as const; +// Has to match `orama-index.ts`, and for a reason that is invisible when it is +// wrong: `load` restores the index data but the tokenizer comes from `create`. +// Build the index with the English stemmer and restore it without, and every +// query term is compared unstemmed against stemmed index terms — `variables` +// found 5 pages instead of 321, and no typo matched at all. Search kept working, +// so nothing failed loudly. +const TOKENIZER = { stemming: true, language: 'english' } as const; + let indexUrl = ''; let ready: Promise | null = null; @@ -36,7 +44,10 @@ function open() { ready ??= fetch(indexUrl) .then((response) => response.json()) .then((raw) => { - const db = create({ schema: SCHEMA }); + const db = create({ + schema: SCHEMA, + components: { tokenizer: TOKENIZER }, + }); load(db, raw); return db; }) From d9ae23977b6cab935817a46ab971675f90cac12a Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Tue, 18 Aug 2026 14:46:25 +1200 Subject: [PATCH 04/11] Drop the sort store, trim the stored body, and add stop words Three changes to what the index carries, none of which change what it can find. Nothing in the engine sorts, but Orama builds and serializes a sort store for every sortable field unless it is turned off. That was 2.69MB. Each document was stored whole, so 2.2MB of page text shipped to every visitor purely so the excerpt could be cut from it client-side. The inverted index is a separate structure and keeps every term, so trimming the stored copy to 200 characters costs no recall. The excerpt window is 180. Orama defaults stopWords to an empty list, so `how`, `do`, `the` and 177 others were live search terms. The list ships separately as @orama/stopwords. Measured against the deployed index this is worth 80% to 87% on natural-language queries. The index goes 11.4MB to 5.3MB raw, and 1.32MB to 0.66MB brotli. That is below the predicted 6.53MB because the stop-word list shrinks the index as well, which had not been measured before. sort and the tokenizer are set in both the integration and the worker, for the same reason: the worker restores into a database it creates itself, so anything the index was built with has to be declared on both sides. Co-Authored-By: Claude Opus 5 (1M context) --- dictionary-octopus.txt | 1 + package.json | 1 + pnpm-lock.yaml | 9 +++++++++ src/integrations/orama-index.ts | 31 +++++++++++++++++++++++++++++-- src/scripts/orama-worker.ts | 12 +++++++++++- 5 files changed, 51 insertions(+), 3 deletions(-) diff --git a/dictionary-octopus.txt b/dictionary-octopus.txt index 7c66dcf4f6..691b6eda1d 100644 --- a/dictionary-octopus.txt +++ b/dictionary-octopus.txt @@ -549,6 +549,7 @@ statefulset statefulsets stepsprodpackages sthumb +stopwords storyblok strconv struct diff --git a/package.json b/package.json index 3c07ba4df1..d415643f9d 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ }, "devDependencies": { "@orama/orama": "^3.1.18", + "@orama/stopwords": "^3.1.18", "@playwright/test": "^1.60.0", "@pnpm/exe": "^11.0.9", "cspell": "^10.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a687f9e09e..58e7d45b60 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -52,6 +52,9 @@ importers: '@orama/orama': specifier: ^3.1.18 version: 3.1.18 + '@orama/stopwords': + specifier: ^3.1.18 + version: 3.1.18 '@playwright/test': specifier: ^1.60.0 version: 1.62.0 @@ -816,6 +819,10 @@ packages: resolution: {integrity: sha512-a61ljmRVVyG5MC/698C8/FfFDw5a8LOIvyOLW5fztgUXqUpc1jOfQzOitSCbge657OgXXThmY3Tk8fpiDb4UcA==} engines: {node: '>= 20.0.0'} + '@orama/stopwords@3.1.18': + resolution: {integrity: sha512-W8V7m7RnCme+99OmKl/xs5rf6OUhFpr0aPGVmPrXzTLSg4ZqSbRY2euS2S/lgjjYi/0NhEWqwoq8nDY6Ihx4EA==} + engines: {node: '>= 20.0.0'} + '@oslojs/encoding@1.1.0': resolution: {integrity: sha512-70wQhgYmndg4GCPxPPxPGevRKqTIJ2Nh4OkiMWmDAVYsTQ+Ta7Sq+rPevXyXGdzr30/qZBnyOalCszoMxlyldQ==} @@ -3766,6 +3773,8 @@ snapshots: '@orama/orama@3.1.18': {} + '@orama/stopwords@3.1.18': {} + '@oslojs/encoding@1.1.0': {} '@oxc-project/types@0.144.0': {} diff --git a/src/integrations/orama-index.ts b/src/integrations/orama-index.ts index d53aefcc06..489b76f086 100644 --- a/src/integrations/orama-index.ts +++ b/src/integrations/orama-index.ts @@ -17,6 +17,9 @@ import { globSync } from 'glob'; // to restore this fails outright. `save` returns a plain object, which is all // the JSON format ever was. import { create, insertMultiple, save } from '@orama/orama'; +// Orama defaults `stopWords` to an empty list, so without this `how`, `do`, +// `the` and 177 others are live search terms. +import { stopwords as englishStopwords } from '@orama/stopwords/english'; // Roughly the number of pages that pass the search predicate. Well under it and // the corpus has moved, which is a silent failure otherwise. @@ -26,6 +29,11 @@ const EXPECTED_PAGES = 1000; // are near the top, and a result excerpt never reaches further than this. const BODY_LIMIT = 4000; +// How much of each page's text is kept in the stored copy. The inverted index +// is a separate structure and keeps every term, so this only bounds the text an +// excerpt can be cut from, and `excerptFrom` never shows more than 180. +const EXCERPT_LIMIT = 200; + const SECTIONS: [string, RegExp][] = [ [ 'cli', @@ -107,17 +115,36 @@ export default function oramaIndex(): AstroIntegration { trail: 'string', section: 'enum', } as const, + // Nothing sorts, and a sort store is built and serialized for every + // sortable field unless it is turned off. It was 2.69MB of the index. + sort: { enabled: false }, // `orama-worker.ts` has to create its database with this same // tokenizer before it restores the index, or query terms are matched // unstemmed against stemmed index terms and recall collapses quietly. components: { - tokenizer: { stemming: true, language: 'english' }, + tokenizer: { + stemming: true, + language: 'english', + stopWords: englishStopwords, + }, }, }); await insertMultiple(db, documents); - const serialized = JSON.stringify(save(db)); + // Reaches into the shape `save()` returns, so an Orama release that + // changes it breaks here rather than silently shipping the whole corpus + // again. Each document is otherwise stored whole, which put 2.2MB of + // page text on the wire purely to cut a 180-character excerpt from. + const stored = save(db) as { + docs: { docs: Record }; + }; + for (const document of Object.values(stored.docs.docs)) { + if (document?.body) + document.body = document.body.slice(0, EXCERPT_LIMIT); + } + + const serialized = JSON.stringify(stored); const target = path.join(docsDir, 'search-index.json'); fs.writeFileSync(target, serialized, 'utf8'); diff --git a/src/scripts/orama-worker.ts b/src/scripts/orama-worker.ts index e108ec7fe2..27576d4fed 100644 --- a/src/scripts/orama-worker.ts +++ b/src/scripts/orama-worker.ts @@ -9,6 +9,7 @@ // worker fails at import time. import { create, load, search } from '@orama/orama'; +import { stopwords as englishStopwords } from '@orama/stopwords/english'; type LoadMessage = { type: 'load'; indexUrl: string }; type SearchMessage = { @@ -35,7 +36,13 @@ const SCHEMA = { // query term is compared unstemmed against stemmed index terms — `variables` // found 5 pages instead of 321, and no typo matched at all. Search kept working, // so nothing failed loudly. -const TOKENIZER = { stemming: true, language: 'english' } as const; +const TOKENIZER = { + stemming: true, + language: 'english', + // Orama defaults this to an empty list, so without it `how`, `do`, `the` and + // 177 others are live search terms on both sides. + stopWords: englishStopwords, +} as const; let indexUrl = ''; let ready: Promise | null = null; @@ -46,6 +53,9 @@ function open() { .then((raw) => { const db = create({ schema: SCHEMA, + // Matches `orama-index.ts`: the index was built without a sort store, + // so restoring into a database that expects one is a mismatch. + sort: { enabled: false }, components: { tokenizer: TOKENIZER }, }); load(db, raw); From 73d4d7b359347f65166b5d790dbe8e2d22f421d3 Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Tue, 18 Aug 2026 14:52:32 +1200 Subject: [PATCH 05/11] Take facet counts from Orama rather than counting the returned hits The engine asked the worker for 200 hits and counted sections over them, so every count saturated: a search for `variables` showed All (200) when the real total is 321. It also filtered those 200 client-side, so a section whose matches fell outside them came back short. The worker now asks for facets, which report the whole match set, and applies the selected section as a `where` clause so the filtered list is complete. Counts come from an unfiltered search on purpose. A `where` clause narrows the facet values to the section being filtered on, and the strip has to keep showing what the other tabs hold. Verified against the built index: All (321), Docs (290), API (19), CLI (12), and selecting CLI returns all 12. Co-Authored-By: Claude Opus 5 (1M context) --- src/scripts/orama-worker.ts | 29 +++++++++++++++++++++++++++-- src/scripts/search-engine-orama.ts | 27 +++++++++++---------------- 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/src/scripts/orama-worker.ts b/src/scripts/orama-worker.ts index 27576d4fed..bf16856c69 100644 --- a/src/scripts/orama-worker.ts +++ b/src/scripts/orama-worker.ts @@ -16,6 +16,7 @@ type SearchMessage = { type: 'search'; id: number; query: string; + facet?: string; limit: number; }; @@ -85,18 +86,42 @@ self.addEventListener( try { const db = await open(); - const results = await search(db as never, { + const query = { term: message.query, limit: message.limit, // Body carries the most text and the least signal per word, so it is // weighted below the fields that name what a page is about. boost: { title: 4, description: 2, trail: 2, body: 1 }, tolerance: 1, + }; + + // Unfiltered, because a `where` clause narrows the facet counts to the + // section being filtered on, and the tab strip has to keep showing what + // the other tabs hold. `count` is the true total rather than the capped + // number of hits returned. + const overview = await search(db as never, { + ...query, + facets: { section: {} }, }); + // Filtering in the worker rather than over the returned hits: a section + // whose matches fall outside the first `limit` would otherwise come back + // short. + const selected = + message.facet && message.facet !== 'all' + ? await search(db as never, { + ...query, + where: { section: { eq: message.facet } }, + }) + : overview; + self.postMessage({ id: message.id, - hits: results.hits.map((hit) => hit.document), + counts: { + all: overview.count, + ...(overview.facets?.section?.values ?? {}), + }, + hits: selected.hits.map((hit) => hit.document), }); } catch (error) { self.postMessage({ id: message.id, error: String(error) }); diff --git a/src/scripts/search-engine-orama.ts b/src/scripts/search-engine-orama.ts index cc2a923908..c465b45b31 100644 --- a/src/scripts/search-engine-orama.ts +++ b/src/scripts/search-engine-orama.ts @@ -10,7 +10,6 @@ import { classify, - countByFacet, type SearchEngine, type SearchResult, } from './search-engine'; @@ -28,6 +27,8 @@ type WorkerHit = { type WorkerReply = { id: number; hits?: WorkerHit[]; + /** Per-section totals across the whole match set, plus `all`. */ + counts?: Record; error?: string; }; @@ -96,18 +97,17 @@ export function oramaEngine(indexUrl: string): SearchEngine { async search(rawQuery, facet) { const query = rawQuery.trim(); - if (!query) return { results: [], counts: countByFacet([]) }; + const empty = { results: [], counts: { all: 0 } }; + if (!query) return empty; - // Counts come from the whole match set rather than the filtered one, so - // the strip keeps showing what the other tabs hold while one is selected. - const reply = await ask(query); - if (reply.error || !reply.hits) { - return { results: [], counts: countByFacet([]) }; - } + // The worker applies the facet and reports the counts, because both need + // the whole match set and it is the only side that has it. + const reply = await ask(query, facet); + if (reply.error || !reply.hits || !reply.counts) return empty; const terms = query.split(/\s+/).filter((term) => term.length > 1); - const all = reply.hits.map((hit): SearchResult => { + const results = reply.hits.map((hit): SearchResult => { const path = new URL(hit.url, window.location.origin).pathname; return { url: path, @@ -118,14 +118,9 @@ export function oramaEngine(indexUrl: string): SearchEngine { }; }); - const shown = - facet && facet !== 'all' - ? all.filter((result) => result.facet === facet) - : all; - return { - results: shown.slice(0, RESULT_LIMIT), - counts: countByFacet(all), + results: results.slice(0, RESULT_LIMIT), + counts: reply.counts, }; }, }; From 5add63aa6eacda7c2e5bfa16b3035347b52ceff8 Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Tue, 18 Aug 2026 15:03:57 +1200 Subject: [PATCH 06/11] Cut the excerpt from the field the query matched, and settle BODY_LIMIT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit excerptFrom preferred the description outright, so any page with a subtitle showed its subtitle even when the match that earned it its rank was in the body. It now picks whichever field the query actually hit, falling back as before. BODY_LIMIT stays at 4,000, now as a measured decision rather than an assumption. Scored against the bake-off query set with a local build: 2,000 4.28MB Success@5 78% intent 87% 4,000 5.33MB Success@5 78% intent 87% 20,000 6.63MB Success@5 76% intent 80% Raising it is worse and bigger — the extra text dilutes the terms that identify a page, which is the opposite of what the audit expected. Lowering it scores the same and saves 1.05MB, but narrows what is findable on long pages to buy bytes the payload budget does not need: brotli is 0.66MB against a 0.81MB gate. Co-Authored-By: Claude Opus 5 (1M context) --- src/integrations/orama-index.ts | 8 ++++++-- src/scripts/search-engine-orama.ts | 11 +++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/integrations/orama-index.ts b/src/integrations/orama-index.ts index 489b76f086..b203a40c68 100644 --- a/src/integrations/orama-index.ts +++ b/src/integrations/orama-index.ts @@ -25,8 +25,12 @@ import { stopwords as englishStopwords } from '@orama/stopwords/english'; // the corpus has moved, which is a silent failure otherwise. const EXPECTED_PAGES = 1000; -// Long pages cost index size for very little relevance — the terms that matter -// are near the top, and a result excerpt never reaches further than this. +// How much of each page is indexed. Measured against the bake-off query set +// rather than assumed: raising this to 20,000 costs 1.3MB and makes relevance +// worse (Success@5 78% to 76%, intent 87% to 80%), because the extra text +// dilutes the terms that identify a page. Lowering it to 2,000 saves 1.05MB and +// scores the same, but narrows what is findable on long pages for no gain the +// payload budget needs — brotli is already 0.66MB against a 0.81MB gate. const BODY_LIMIT = 4000; // How much of each page's text is kept in the stored copy. The inverted index diff --git a/src/scripts/search-engine-orama.ts b/src/scripts/search-engine-orama.ts index c465b45b31..c846f64606 100644 --- a/src/scripts/search-engine-orama.ts +++ b/src/scripts/search-engine-orama.ts @@ -49,10 +49,17 @@ function escapeForRegExp(term: string) { * has to cut its own. */ function excerptFrom(hit: WorkerHit, terms: string[]) { - const source = hit.description || hit.body; + const pattern = new RegExp(terms.map(escapeForRegExp).join('|'), 'i'); + + // Whichever field the query actually hit. Preferring the description outright + // showed every page with a subtitle its subtitle, even when the match that + // earned it its rank was in the body. + const source = + [hit.body, hit.description].find((text) => text && pattern.test(text)) ?? + hit.description ?? + hit.body; if (!source) return ''; - const pattern = new RegExp(terms.map(escapeForRegExp).join('|'), 'i'); const at = source.search(pattern); const from = at > 60 ? source.lastIndexOf(' ', at - 60) + 1 : 0; const window = source.slice(from, from + 180); From 48b2a58eb4faf0397e2d0e3fbf6c20863cfde1ba Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Tue, 18 Aug 2026 18:03:15 +1200 Subject: [PATCH 07/11] Stop the excerpt falling back to an empty description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 26% of top-5 rows rendered with a blank excerpt — 127 of 482 across the query set. Two things compounded, and the first was mine: `?? hit.description` treats an empty string as a value worth keeping, so it won the fallback and blocked `hit.body`. 1,200 of the 1,254 documents have an empty description and none have an empty body, so that path was taken almost every time it was reached. It was reached far more often than it looks, because the index matches stemmed terms and the excerpt matches literally. "Guided failures" legitimately ranks a page whose text only ever says "guides" — both stem to "guid" — and the literal regex then finds nothing, falling straight into the trap above. `||` instead, with body first. When nothing matches literally the window opens at the start of the page, which is what a reader wants from a stemmed match; it just arrives unhighlighted. Trimming the stored body to 200 characters did not cause this. At 4,000 a literal match usually turned up somewhere by luck, so it hid the bug. Measured against the same 102 queries: blank rows 127/482 to 0/482, and queries with a blank in the top five 50/102 to 0/102. The highlightable rate is unchanged at 72%, which is the point — this replaces blanks with real text rather than inventing highlights for stemmed matches. Co-Authored-By: Claude Opus 5 (1M context) --- src/scripts/search-engine-orama.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/scripts/search-engine-orama.ts b/src/scripts/search-engine-orama.ts index c846f64606..b76d41eb4a 100644 --- a/src/scripts/search-engine-orama.ts +++ b/src/scripts/search-engine-orama.ts @@ -54,10 +54,19 @@ function excerptFrom(hit: WorkerHit, terms: string[]) { // Whichever field the query actually hit. Preferring the description outright // showed every page with a subtitle its subtitle, even when the match that // earned it its rank was in the body. - const source = - [hit.body, hit.description].find((text) => text && pattern.test(text)) ?? - hit.description ?? - hit.body; + // + // `||` rather than `??` on the fallback, and body before description: 1,200 of + // the 1,254 documents have an empty description, and `??` treats "" as a value + // to keep, so it won every time and blocked the fallback to body. + // + // The fallback runs more often than it looks. The index matches stemmed terms + // and this matches literally, so "guided failures" legitimately ranks a page + // whose text only ever says "guides" — the regex then finds nothing. Opening + // the page text is the right answer there; it just arrives unhighlighted. + const matched = [hit.body, hit.description].find( + (text) => text && pattern.test(text) + ); + const source = matched || hit.body || hit.description; if (!source) return ''; const at = source.search(pattern); From 58fe5e19affa47f2b411ef306ce99dc7e440fc28 Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Tue, 18 Aug 2026 19:28:12 +1200 Subject: [PATCH 08/11] Start loading the index when someone reaches for the search field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `warm()` only ran when the overlay opened, so the whole index was fetched and parsed while the reader waited. It now starts on the first pointer or focus reaching a search field, which costs nothing for the majority who never go near it. `SearchEngine` carries the policy rather than the overlay assuming one, because the right answer differs by engine: an engine whose runtime is small enough can warm on page load, and this one cannot — 5.3MB spent on every visitor, searcher or not. Ctrl/Cmd+K goes straight to open, which still warms, so the policy only decides how much earlier it can start. Measured against a local build on one server, so caching is held constant: 778ms to first result without the hover, 646ms with. The saving is smaller than the load cost because the load is not what dominates — roughly 500ms of that 646ms is the query itself, which is worth looking at separately now that facet counts are computed over the whole match set. Co-Authored-By: Claude Opus 5 (1M context) --- src/scripts/docs-search.ts | 18 ++++++++++++++++++ src/scripts/search-engine-orama.ts | 5 +++++ src/scripts/search-engine.ts | 15 +++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/src/scripts/docs-search.ts b/src/scripts/docs-search.ts index c22dc25e14..4d0423d586 100644 --- a/src/scripts/docs-search.ts +++ b/src/scripts/docs-search.ts @@ -311,6 +311,22 @@ function setup(dialog: HTMLDialogElement) { } }); + // Reaching for the field is the earliest honest signal that someone means to + // search. Once only — `warm` is idempotent, but there is no reason to keep + // asking. + if (engine.warmOn === 'intent') { + let warmed = false; + const onIntent = (event: Event) => { + if (warmed || !opensThis(event.target)) return; + warmed = true; + engine.warm?.(); + }; + // `pointerover` rather than `pointerenter`, which does not bubble to a + // delegated listener, and `focusin` for the keyboard path. + document.addEventListener('pointerover', onIntent); + document.addEventListener('focusin', onIntent); + } + // Chords and shared links belong to the page's own search, not to a demo // instance sitting in an article. if (name !== SITE_SEARCH) return; @@ -328,6 +344,8 @@ function setup(dialog: HTMLDialogElement) { open(); }); + if (engine.warmOn === 'load') engine.warm?.(); + // A shared `?q=` link opens straight into results. const q = new URLSearchParams(window.location.search).get('q'); if (q) open(q); diff --git a/src/scripts/search-engine-orama.ts b/src/scripts/search-engine-orama.ts index b76d41eb4a..7e52094eac 100644 --- a/src/scripts/search-engine-orama.ts +++ b/src/scripts/search-engine-orama.ts @@ -109,6 +109,11 @@ export function oramaEngine(indexUrl: string): SearchEngine { } return { + // Not on page load: the index is 5.3MB, and warming it for every visitor + // spends that on the majority who never search. On hover the bytes are + // usually already cached, so the 40-67ms parse finishes before the click + // does. + warmOn: 'intent', warm: load, async search(rawQuery, facet) { diff --git a/src/scripts/search-engine.ts b/src/scripts/search-engine.ts index c8d1800b2b..feb919f03d 100644 --- a/src/scripts/search-engine.ts +++ b/src/scripts/search-engine.ts @@ -22,6 +22,21 @@ export type SearchEngine = { search(query: string, facet?: string): Promise; /** Optional: start loading the index before the first query needs it. */ warm?(): void; + /** + * When `warm()` is worth running, which depends on what the engine has to + * fetch and parse: + * + * - `load` — on page load. For an engine whose runtime is small enough that + * every visitor can pay for it, searcher or not. + * - `intent` — the first time the pointer or focus reaches a search field. + * Costs nothing for the majority who never go near it, and buys the + * 200-500ms between reaching for the field and clicking it. + * - `open` — not until the overlay opens. The default. + * + * Ctrl/Cmd+K skips straight to `open`, so warming still happens there; this + * only decides how much earlier it can start. + */ + warmOn?: 'load' | 'intent' | 'open'; }; export type Facet = { key: string; label: string }; From eece8016cc3fdb231f4e02df41c28cb41fff4393 Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Wed, 19 Aug 2026 08:49:30 +1200 Subject: [PATCH 09/11] Keep the under-construction API reference out of the index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Main published 106 generated pages under docs/api and deliberately kept them out of search and the sitemap until the section has a landing page. That check lived in search.json.ts, which this branch deletes, so it had to move with the index rather than be lost with it. It does not come along on its own. The Orama index is built from what llm-md-emitter writes, and the emitter's eligibility check knows nothing about the section — those pages carry a layout and no `navSitemap: false`, so they pass. The emitter now writes 1,360 files where it wrote 1,254, and all 106 of the difference would have been searchable. Indexed count is back to 1,254, with nothing under /docs/api/ in the index. Co-Authored-By: Claude Opus 5 (1M context) --- src/integrations/orama-index.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/integrations/orama-index.ts b/src/integrations/orama-index.ts index b203a40c68..9627ef4740 100644 --- a/src/integrations/orama-index.ts +++ b/src/integrations/orama-index.ts @@ -17,6 +17,10 @@ import { globSync } from 'glob'; // to restore this fails outright. `save` returns a plain object, which is all // the JSON format ever was. import { create, insertMultiple, save } from '@orama/orama'; +// The generated API reference is published but deliberately unsearchable until +// the section has a landing page. `search.json.ts` applied this and is deleted +// here, so the check has to move with the index rather than be lost with it. +import { isUnderConstruction } from '../lib/underConstruction'; // Orama defaults `stopWords` to an empty list, so without this `how`, `do`, // `the` and 177 others are live search terms. import { stopwords as englishStopwords } from '@orama/stopwords/english'; @@ -84,11 +88,14 @@ export default function oramaIndex(): AstroIntegration { const distDir = fileURLToPath(dir); const docsDir = path.join(distDir, 'docs'); + // The emitter writes every page that passes its own eligibility check, + // which does not know about the under-construction section — those pages + // have a layout and no `navSitemap: false`, so they come through. const files = globSync('**/*.md', { cwd: docsDir, nodir: true, posix: true, - }); + }).filter((file) => !isUnderConstruction(file)); const documents = files.map((file) => { const raw = fs From c8f96187e11515b4b90d6b6764d18b7803f9c720 Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Wed, 19 Aug 2026 09:26:10 +1200 Subject: [PATCH 10/11] Rank a section's own page above the pages inside it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BM25 has no notion of a site's shape, so `variables` returned projects/variables/system-variables and `tentacle` returned tentacle/linux: a child page repeats the term more often in less text. On the search terms readers actually type this was the largest single source of missed traffic. Two changes. The title boost goes from 4 to 8, worth 26 points of Success@5 on its own, since most searches are one or two words naming a page. Then hits are reordered so a page the query names outright comes first, and everything else is scored with a penalty per path segment. Measured against three query sets — real search terms, top-visited pages, and a curated set — rather than only the one it was tuned on. On real search terms Success@5 goes 54% to 80% and rank one 17% to 51%. Deep pages still rank first when they are what was asked for. A tiebreak on depth was tried first and did nothing: BM25 scores almost never tie, so depth has to scale the score. Co-Authored-By: Claude Opus 5 (1M context) --- src/scripts/orama-worker.ts | 71 +++++++++++++++++++++++++++++++++++-- 1 file changed, 68 insertions(+), 3 deletions(-) diff --git a/src/scripts/orama-worker.ts b/src/scripts/orama-worker.ts index bf16856c69..dd0f6ba9ee 100644 --- a/src/scripts/orama-worker.ts +++ b/src/scripts/orama-worker.ts @@ -45,6 +45,67 @@ const TOKENIZER = { stopWords: englishStopwords, } as const; +// How much a page's depth counts against it, per path segment. BM25 has no +// notion of a site's shape, so on its own it ranks `/docs/projects/variables` +// below `/docs/projects/variables/system-variables` for the query `variables`: +// the child page repeats the term more often in less text. Readers searching a +// bare section name almost always want the section, and on real search logs this +// single failure was the largest source of missed traffic. +// +// 0.12 was picked by sweeping it against three query sets — the search terms +// readers actually typed, the pages they actually visit, and a curated set. Much +// lower and landing pages stay buried; much higher and a genuinely deep page +// cannot win even when it is the only match. +const DEPTH_PENALTY = 0.12; + +type Ranked = { score: number; document: { url: string; title: string } }; + +/** Lowercased, punctuation collapsed, so `Config as Code` matches `config as code`. */ +function comparable(text: string) { + return text + .toLowerCase() + .replace(/[^a-z0-9 ]+/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +function segments(url: string) { + return url.split('/').filter(Boolean); +} + +/** + * Reorders hits so a section's own page beats the pages inside it. + * + * Two signals, in order. A page the query *names* — by title or by the last + * segment of its URL — wins outright, which is what makes `runbooks` return + * `/docs/runbooks`. Everything else is ordered by score discounted for depth. + * + * A plain tiebreak on depth was tried first and did nothing: BM25 scores are + * floats and almost never tie, so depth has to scale the score rather than break + * a draw between equal ones. + */ +function byNameThenDepth(hits: T[], term: string): T[] { + const wanted = comparable(term); + + const names = (hit: T) => { + const slug = segments(hit.document.url).pop() ?? ''; + return comparable(hit.document.title) === wanted || + comparable(slug.replace(/-/g, ' ')) === wanted + ? 1 + : 0; + }; + + return hits + .map((hit) => ({ + hit, + names: names(hit), + adjusted: + hit.score / (1 + DEPTH_PENALTY * segments(hit.document.url).length), + })) + .sort((a, b) => b.names - a.names || b.adjusted - a.adjusted) + .map((entry) => entry.hit); +} + let indexUrl = ''; let ready: Promise | null = null; @@ -90,8 +151,10 @@ self.addEventListener( term: message.query, limit: message.limit, // Body carries the most text and the least signal per word, so it is - // weighted below the fields that name what a page is about. - boost: { title: 4, description: 2, trail: 2, body: 1 }, + // weighted below the fields that name what a page is about. Title at 8 + // rather than 4 was worth 26 points of Success@5 on real search terms on + // its own — most searches are one or two words that name a page. + boost: { title: 8, description: 2, trail: 2, body: 1 }, tolerance: 1, }; @@ -121,7 +184,9 @@ self.addEventListener( all: overview.count, ...(overview.facets?.section?.values ?? {}), }, - hits: selected.hits.map((hit) => hit.document), + hits: byNameThenDepth(selected.hits, message.query).map( + (hit) => hit.document + ), }); } catch (error) { self.postMessage({ id: message.id, error: String(error) }); From 438570fc5fcc2a8759db18f81c1a1fa61d43802f Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Wed, 19 Aug 2026 11:09:04 +1200 Subject: [PATCH 11/11] Ask the overlay whether the API reference is findable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test read /docs/search.json, which this branch deletes along with the rest of the legacy engine, so it was parsing a 404 page as JSON. It now drives the search overlay instead. The two spikes ship indexes of different shapes — one JSON document, one directory of compressed chunks — and neither is readable the way the old one was, but what has to hold is the same either way: a reader searching a word the API reference is full of must not be sent into it. `accounts` names pages both inside the API reference and outside it, so the result list is never empty. A query that matched nothing would pass without proving anything. Co-Authored-By: Claude Opus 5 (1M context) --- tests/api-page.spec.ts | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/api-page.spec.ts b/tests/api-page.spec.ts index acfc11d5f6..e30fa3c53d 100644 --- a/tests/api-page.spec.ts +++ b/tests/api-page.spec.ts @@ -60,14 +60,34 @@ test.describe('api page chrome', () => { }); test('the section is kept out of the search index and the sitemap', async ({ + page, request, }) => { // Temporary, alongside src/lib/underConstruction.ts - delete with it when // the API reference goes live. - const index = await (await request.get('/docs/search.json')).json(); - expect(index.filter((entry) => entry.url.includes('/docs/api/'))).toEqual( - [] + // + // Asked through the overlay rather than of an index file. This used to read + // /docs/search.json, which no longer exists, and the two search engines ship + // indexes of different shapes — one JSON, one a directory of compressed + // chunks. What has to hold is the same either way: a reader searching a word + // the API reference is full of must not be sent into it. + await page.goto('/docs/'); + await page + .locator('input[data-docs-search-trigger]:visible') + .first() + .click(); + await page.locator('[data-docs-search-input]').fill('accounts'); + + // `accounts` is deliberate: it names pages in the API reference *and* pages + // outside it, so the list is never empty. A query that matched nothing would + // pass this test without proving anything. + const results = page.locator('[data-docs-search-results] [role="option"]'); + await expect(results.first()).toBeVisible({ timeout: 30_000 }); + + const hrefs = await results.evaluateAll((rows) => + rows.map((row) => row.getAttribute('href') ?? '') ); + expect(hrefs.filter((href) => href.includes('/docs/api/'))).toEqual([]); const sitemap = await (await request.get('/docs/sitemap.xml')).text(); expect(sitemap).not.toContain('/docs/api/');