diff --git a/astro.config.mjs b/astro.config.mjs index a5f2c67a89..a96b699e39 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -3,6 +3,7 @@ import { satteri } from '@astrojs/markdown-satteri'; import mdx from '@astrojs/mdx'; import { attributeMarkdown, wrapTables } from '/src/themes/octopus/utilities/custom-markdown.mjs'; import llmMdEmitter from './src/integrations/llm-md-emitter.ts'; +import oramaIndex from './src/integrations/orama-index.ts'; import pruneDist from './src/integrations/prune-dist.ts'; import satteriHeadingId from './src/plugins/satteri-heading-id.js'; import satteriApiExamples, { apiExampleDirective } from './src/plugins/satteri-api-examples.js'; @@ -22,6 +23,9 @@ export default defineConfig({ integrations: [ mdx(), llmMdEmitter(), + // Indexes the markdown the emitter above just wrote, so it has to come + // after it and before the prune that would delete its output + oramaIndex(), // Must run last: strips build output that can't be served under /docs/ pruneDist() ], diff --git a/dictionary-octopus.txt b/dictionary-octopus.txt index cdcca35cbf..691b6eda1d 100644 --- a/dictionary-octopus.txt +++ b/dictionary-octopus.txt @@ -399,6 +399,7 @@ onlylogs onmicrosoft openon operationalize +orama outfile outputformat OWASP @@ -548,6 +549,7 @@ statefulset statefulsets stepsprodpackages sthumb +stopwords storyblok strconv struct diff --git a/package.json b/package.json index 8ea81c1723..d415643f9d 100644 --- a/package.json +++ b/package.json @@ -33,13 +33,13 @@ "astro-accelerator-utils": "^0.3.84", "glob": "^13.0.6", "gray-matter": "^4.0.3", - "html-to-text": "^10.0.0", - "keyword-extractor": "^0.0.28", "optional": "^0.1.4", "satteri": "^0.9.5", "sharp": "^0.34.5" }, "devDependencies": { + "@orama/orama": "^3.1.18", + "@orama/stopwords": "^3.1.18", "@playwright/test": "^1.60.0", "@pnpm/exe": "^11.0.9", "cspell": "^10.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 715655b317..58e7d45b60 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -39,12 +39,6 @@ importers: gray-matter: specifier: ^4.0.3 version: 4.0.3 - html-to-text: - specifier: ^10.0.0 - version: 10.0.0 - keyword-extractor: - specifier: ^0.0.28 - version: 0.0.28 optional: specifier: ^0.1.4 version: 0.1.4 @@ -55,6 +49,12 @@ importers: specifier: ^0.34.5 version: 0.34.5 devDependencies: + '@orama/orama': + specifier: ^3.1.18 + version: 3.1.18 + '@orama/stopwords': + specifier: ^3.1.18 + version: 3.1.18 '@playwright/test': specifier: ^1.60.0 version: 1.62.0 @@ -815,6 +815,14 @@ packages: '@octopusdeploy/design-system-tokens@2026.3.8240': resolution: {integrity: sha512-SxabL5Y2bDD0kEEEtVUtTos3tiJ+2FEWTno1a40YVw/Y5eUbRnXUTChf4oVsRzw32GXpjBhxoTxiOBdOceeelA==} + '@orama/orama@3.1.18': + resolution: {integrity: sha512-a61ljmRVVyG5MC/698C8/FfFDw5a8LOIvyOLW5fztgUXqUpc1jOfQzOitSCbge657OgXXThmY3Tk8fpiDb4UcA==} + engines: {node: '>= 20.0.0'} + + '@orama/stopwords@3.1.18': + resolution: {integrity: sha512-W8V7m7RnCme+99OmKl/xs5rf6OUhFpr0aPGVmPrXzTLSg4ZqSbRY2euS2S/lgjjYi/0NhEWqwoq8nDY6Ihx4EA==} + engines: {node: '>= 20.0.0'} + '@oslojs/encoding@1.1.0': resolution: {integrity: sha512-70wQhgYmndg4GCPxPPxPGevRKqTIJ2Nh4OkiMWmDAVYsTQ+Ta7Sq+rPevXyXGdzr30/qZBnyOalCszoMxlyldQ==} @@ -1008,11 +1016,6 @@ packages: '@rolldown/pluginutils@1.0.1': resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} - '@selderee/plugin-htmlparser2@0.12.0': - resolution: {integrity: sha512-oELmoyA6ML9jDRMV3kgcMQFKxUfBU0yFVn6yTctVaLT5ygXnxH52I3TZEgV9EhXJC68/uFvE5Daj1/25c0Xa/A==} - peerDependencies: - selderee: ~0.12.0 - '@shikijs/core@4.4.3': resolution: {integrity: sha512-QCR4q2ZO/ILJEuwiBMel4wdcTDb1JGwfjKTxPDF6x8ixOaluPrVqIn06C99AcRPhmYlBR56d/Fb+GN58GzExpg==} engines: {node: '>=20'} @@ -1398,10 +1401,6 @@ packages: decode-named-character-reference@1.3.0: resolution: {integrity: sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==} - deepmerge-ts@7.1.5: - resolution: {integrity: sha512-HOJkrhaYsweh+W+e74Yn7YStZOilkoPb6fycpwNLKzSPtruFs48nYis0zy5yJz1+ktUhHxoRDJ27RQAWLIJVJw==} - engines: {node: '>=16.0.0'} - define-data-property@1.1.4: resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} engines: {node: '>= 0.4'} @@ -1778,10 +1777,6 @@ packages: html-escaper@3.0.3: resolution: {integrity: sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==} - html-to-text@10.0.0: - resolution: {integrity: sha512-2OH59Gtprdczel+7Rxgpz9hGVJREaf8Lt1H4kZwWHpEn70VQKRuMNGsb2eDbwaTzrYzb0hheiOG1P7Dim0B4dQ==} - engines: {node: '>=20.19.0'} - html-void-elements@3.0.0: resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} @@ -1979,17 +1974,10 @@ packages: jsonc-parser@3.3.1: resolution: {integrity: sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==} - keyword-extractor@0.0.28: - resolution: {integrity: sha512-oi7dSPpYtW/3fE0vZiqQgZ8mW3F1V9K4+rBJ0FcVrdXBEQuhZ0zKj7sX74eqGASuepLHf9aYdeonyKHWhYpHQA==} - engines: {node: '>= 0.10.0'} - kind-of@6.0.3: resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} engines: {node: '>=0.10.0'} - leac@0.7.0: - resolution: {integrity: sha512-qMrZeyEekgdRQ9o6a4NAB2EQZrv827GJdn1vnapwSJ90hWRB4TzUSunvacPkxQ2TnNqHNI1/zSt0hlo0crG8Jw==} - lightningcss-android-arm64@1.33.0: resolution: {integrity: sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==} engines: {node: '>= 12.0.0'} @@ -2405,9 +2393,6 @@ packages: parse5@7.3.0: resolution: {integrity: sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==} - parseley@0.13.1: - resolution: {integrity: sha512-uNBJZzmb60l6p6VWLTmevizNAGnE0xoSf1n0B4q3ntegDNzcS68NRCcBDZTcyXHxt2XhBChsCuqj4M+nChvE/A==} - path-key@2.0.1: resolution: {integrity: sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==} engines: {node: '>=4'} @@ -2431,9 +2416,6 @@ packages: resolution: {integrity: sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==} engines: {node: '>=4'} - peberminta@0.10.0: - resolution: {integrity: sha512-80B2AsU+I4Qdb0ZAPSfe9UwvGzwkM37IKIFEvdS3D/3Ndgv2bsuJ0bfG1+iEYO+l7Gfd4EUJmuRyq7efLgRMzQ==} - piccolore@0.1.3: resolution: {integrity: sha512-o8bTeDWjE086iwKrROaDf31K0qC/BENdm15/uH9usSC/uZjJOKb2YGiVHfLY4GhwsERiPI1jmwI2XrA7ACOxVw==} @@ -2634,9 +2616,6 @@ packages: resolution: {integrity: sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==} engines: {node: '>=4'} - selderee@0.12.0: - resolution: {integrity: sha512-b1YMh3+DHZp59DLna3qVwQ5iOla/nrI6mLBNW02XxU77M3046Df6VLkoaJyFz20VsGIG5kkp+FK0kg4K4HnUFw==} - semver@5.7.2: resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} hasBin: true @@ -3792,6 +3771,10 @@ snapshots: '@octopusdeploy/design-system-tokens@2026.3.8240': {} + '@orama/orama@3.1.18': {} + + '@orama/stopwords@3.1.18': {} + '@oslojs/encoding@1.1.0': {} '@oxc-project/types@0.144.0': {} @@ -3916,12 +3899,6 @@ snapshots: '@rolldown/pluginutils@1.0.1': {} - '@selderee/plugin-htmlparser2@0.12.0(selderee@0.12.0)': - dependencies: - domelementtype: 2.3.0 - domhandler: 5.0.3 - selderee: 0.12.0 - '@shikijs/core@4.4.3': dependencies: '@shikijs/primitive': 4.4.3 @@ -4437,8 +4414,6 @@ snapshots: dependencies: character-entities: 2.0.2 - deepmerge-ts@7.1.5: {} - define-data-property@1.1.4: dependencies: es-define-property: 1.0.1 @@ -4998,14 +4973,6 @@ snapshots: html-escaper@3.0.3: {} - html-to-text@10.0.0: - dependencies: - '@selderee/plugin-htmlparser2': 0.12.0(selderee@0.12.0) - deepmerge-ts: 7.1.5 - dom-serializer: 2.0.0 - htmlparser2: 10.1.0 - selderee: 0.12.0 - html-void-elements@3.0.0: {} htmlparser2@10.1.0: @@ -5193,12 +5160,8 @@ snapshots: jsonc-parser@3.3.1: {} - keyword-extractor@0.0.28: {} - kind-of@6.0.3: {} - leac@0.7.0: {} - lightningcss-android-arm64@1.33.0: optional: true @@ -5879,11 +5842,6 @@ snapshots: dependencies: entities: 6.0.1 - parseley@0.13.1: - dependencies: - leac: 0.7.0 - peberminta: 0.10.0 - path-key@2.0.1: {} path-key@3.1.1: {} @@ -5904,8 +5862,6 @@ snapshots: dependencies: pify: 3.0.0 - peberminta@0.10.0: {} - piccolore@0.1.3: {} picocolors@1.1.1: {} @@ -6198,10 +6154,6 @@ snapshots: extend-shallow: 2.0.1 kind-of: 6.0.3 - selderee@0.12.0: - dependencies: - parseley: 0.13.1 - semver@5.7.2: {} semver@7.8.0: {} diff --git a/src/components/DocsSearch.astro b/src/components/DocsSearch.astro index 4957e1b73c..c09bbaf793 100644 --- a/src/components/DocsSearch.astro +++ b/src/components/DocsSearch.astro @@ -44,7 +44,7 @@ if (facets.length === 0) { throw new Error('DocsSearch needs at least one facet for its tab strip'); } -const indexUrl = `${SITE.subfolder}/search.json`; +const indexUrl = `${SITE.subfolder}/search-index.json`; const listboxId = `${name}-search-listbox`; --- diff --git a/src/integrations/orama-index.ts b/src/integrations/orama-index.ts new file mode 100644 index 0000000000..9627ef4740 --- /dev/null +++ b/src/integrations/orama-index.ts @@ -0,0 +1,175 @@ +// Builds an Orama index and serializes it for the browser to restore. +// +// The corpus is the `.md` files `llm-md-emitter.ts` has already written into +// `dist/docs/`. That emitter runs the same eligibility predicate search uses, so +// redirect stubs and `navSearch: false` pages are already gone and there is no +// second extraction pipeline to keep in step with the first. This has to be +// registered after `llmMdEmitter()` and before `pruneDist()`, which would delete +// anything written outside `dist/docs/`. + +import type { AstroIntegration } from 'astro'; +import { fileURLToPath } from 'node:url'; +import * as path from 'node:path'; +import * as fs from 'node:fs'; +import { globSync } from 'glob'; +// Core `save` rather than @orama/plugin-data-persistence: the plugin reaches for +// Node's filesystem and buffers, and bundling it for the browser worker that has +// to restore this fails outright. `save` returns a plain object, which is all +// the JSON format ever was. +import { create, insertMultiple, save } from '@orama/orama'; +// The generated API reference is published but deliberately unsearchable until +// the section has a landing page. `search.json.ts` applied this and is deleted +// here, so the check has to move with the index rather than be lost with it. +import { isUnderConstruction } from '../lib/underConstruction'; +// Orama defaults `stopWords` to an empty list, so without this `how`, `do`, +// `the` and 177 others are live search terms. +import { stopwords as englishStopwords } from '@orama/stopwords/english'; + +// Roughly the number of pages that pass the search predicate. Well under it and +// the corpus has moved, which is a silent failure otherwise. +const EXPECTED_PAGES = 1000; + +// How much of each page is indexed. Measured against the bake-off query set +// rather than assumed: raising this to 20,000 costs 1.3MB and makes relevance +// worse (Success@5 78% to 76%, intent 87% to 80%), because the extra text +// dilutes the terms that identify a page. Lowering it to 2,000 saves 1.05MB and +// scores the same, but narrows what is findable on long pages for no gain the +// payload budget needs — brotli is already 0.66MB against a 0.81MB gate. +const BODY_LIMIT = 4000; + +// How much of each page's text is kept in the stored copy. The inverted index +// is a separate structure and keeps every term, so this only bounds the text an +// excerpt can be cut from, and `excerptFrom` never shows more than 180. +const EXCERPT_LIMIT = 200; + +const SECTIONS: [string, RegExp][] = [ + [ + 'cli', + /^\/docs\/octopus-rest-api\/(cli|octopus-cli|[a-z.]+-command-line)(\/|$)/, + ], + ['api', /^\/docs\/octopus-rest-api(\/|$)/], + ['integrations', /^\/docs\/api-and-integration(\/|$)/], +]; + +function sectionFor(url: string) { + return SECTIONS.find(([, prefix]) => prefix.test(url))?.[0] ?? 'docs'; +} + +function trailFor(url: string) { + return url + .split('/') + .filter(Boolean) + .slice(1, -1) + .map((segment) => + segment.replace(/-/g, ' ').replace(/^./, (c) => c.toUpperCase()) + ) + .join(' / '); +} + +/** Strips the markup that would otherwise be indexed as words in its own right: + * fences, link syntax, heading markers and emphasis. */ +function toPlainText(markdown: string) { + return markdown + .replace(/```[\s\S]*?```/g, ' ') + .replace(/`([^`]*)`/g, '$1') + .replace(/!\[[^\]]*\]\([^)]*\)/g, ' ') + .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1') + .replace(/^[#>\s-]+/gm, ' ') + .replace(/[*_]{1,3}/g, '') + .replace(/\s+/g, ' ') + .trim(); +} + +export default function oramaIndex(): AstroIntegration { + return { + name: 'orama-index', + hooks: { + 'astro:build:done': async ({ dir, logger }) => { + const distDir = fileURLToPath(dir); + const docsDir = path.join(distDir, 'docs'); + + // The emitter writes every page that passes its own eligibility check, + // which does not know about the under-construction section — those pages + // have a layout and no `navSitemap: false`, so they come through. + const files = globSync('**/*.md', { + cwd: docsDir, + nodir: true, + posix: true, + }).filter((file) => !isUnderConstruction(file)); + + const documents = files.map((file) => { + const raw = fs + .readFileSync(path.join(docsDir, file), 'utf8') + .replace(/^/, ''); + + // The emitter writes `# Title`, an optional `> subtitle`, then body. + const [, title = ''] = raw.match(/^#\s+(.*)$/m) ?? []; + const [, subtitle = ''] = raw.match(/^>\s+(.*)$/m) ?? []; + const url = '/docs/' + file.replace(/\.md$/, ''); + + return { + url, + title, + description: subtitle, + body: toPlainText(raw).slice(0, BODY_LIMIT), + trail: trailFor(url), + section: sectionFor(url), + }; + }); + + const db = create({ + schema: { + url: 'string', + title: 'string', + description: 'string', + body: 'string', + trail: 'string', + section: 'enum', + } as const, + // Nothing sorts, and a sort store is built and serialized for every + // sortable field unless it is turned off. It was 2.69MB of the index. + sort: { enabled: false }, + // `orama-worker.ts` has to create its database with this same + // tokenizer before it restores the index, or query terms are matched + // unstemmed against stemmed index terms and recall collapses quietly. + components: { + tokenizer: { + stemming: true, + language: 'english', + stopWords: englishStopwords, + }, + }, + }); + + await insertMultiple(db, documents); + + // Reaches into the shape `save()` returns, so an Orama release that + // changes it breaks here rather than silently shipping the whole corpus + // again. Each document is otherwise stored whole, which put 2.2MB of + // page text on the wire purely to cut a 180-character excerpt from. + const stored = save(db) as { + docs: { docs: Record }; + }; + for (const document of Object.values(stored.docs.docs)) { + if (document?.body) + document.body = document.body.slice(0, EXCERPT_LIMIT); + } + + const serialized = JSON.stringify(stored); + const target = path.join(docsDir, 'search-index.json'); + fs.writeFileSync(target, serialized, 'utf8'); + + const megabytes = (serialized.length / 1024 / 1024).toFixed(1); + logger.info( + `indexed ${documents.length} pages into docs/search-index.json (${megabytes}MB)` + ); + + if (documents.length < EXPECTED_PAGES) { + logger.warn( + `only ${documents.length} pages were indexed, expected at least ${EXPECTED_PAGES} — check llm-md-emitter still runs before this` + ); + } + }, + }, + }; +} diff --git a/src/pages/docs/search.json.ts b/src/pages/docs/search.json.ts deleted file mode 100644 index 6ef6c53a15..0000000000 --- a/src/pages/docs/search.json.ts +++ /dev/null @@ -1,93 +0,0 @@ -/** @format */ - -// warning: This file is overwritten by Astro Accelerator - -import { accelerator } from '@lib/accelerator'; -import { PostFiltering } from 'astro-accelerator-utils'; -import type { MarkdownInstance } from 'astro'; -import { SITE } from '@config'; -import { convert } from 'html-to-text'; -import keywordExtractor from 'keyword-extractor'; -import { isUnderConstruction } from '@lib/underConstruction'; - -const getData = async () => { - //@ts-ignore - const allPages = import.meta.glob(['./**/*.md', './**/*.mdx']); - const items = []; - - for (const path in allPages) { - // Temporary - see src/lib/underConstruction.ts. - if (isUnderConstruction(path)) { - continue; - } - - const page = (await allPages[path]()) as MarkdownInstance< - Record - >; - - if (!PostFiltering.showInSearch(page)) { - continue; - } - - let url = page.url ?? ''; - - if (page.frontmatter.paged) { - url += '/1/'; - } - - const headings = await page.getHeadings(); - const title = await accelerator.markdown.getTextFrom( - page.frontmatter?.title - ); - const content = page.compiledContent ? await page.compiledContent() : ''; - let counted: { word: string; count: number }[] = []; - - if (content) { - const options = { - wordwrap: false, - selectors: [ - { selector: 'a', options: { ignoreHref: true } }, - { selector: 'img', format: 'skip' }, - { selector: 'h1', options: { uppercase: false } }, - ], - }; - const text = convert(content, options); - - const words = keywordExtractor.extract(text, { - language: 'english', - return_changed_case: true, - remove_duplicates: true, - }); - - counted = words - .map((w) => { - return { - word: w, - count: words.filter((wd) => wd === w).length, - }; - }) - .filter((e) => e.word.replace(/[^a-z]+/g, '').length > 1); - } - - items.push({ - title: title, - headings: headings.map((h) => { - return { text: h.text, slug: h.slug }; - }), - description: page.frontmatter.description ?? '', - keywords: counted.map((c) => c.word).join(' '), - tags: page.frontmatter.tags ?? [], - url: SITE.url + accelerator.urlFormatter.formatAddress(url), - date: page.frontmatter.pubDate ?? '', - }); - } - - return new Response(JSON.stringify(items), { - status: 200, - headers: { - 'Content-Type': 'application/json', - }, - }); -}; - -export const GET = getData; diff --git a/src/scripts/docs-search.ts b/src/scripts/docs-search.ts index ef88eb5758..4d0423d586 100644 --- a/src/scripts/docs-search.ts +++ b/src/scripts/docs-search.ts @@ -14,7 +14,7 @@ import { type SearchEngine, type SearchResult, } from './search-engine'; -import { legacyEngine } from './search-engine-legacy'; +import { oramaEngine } from './search-engine-orama'; const DEBOUNCE_MS = 150; const SITE_SEARCH = 'site'; @@ -41,7 +41,7 @@ function setup(dialog: HTMLDialogElement) { const demo = dialog.dataset.demoResults; const engine: SearchEngine = demo ? fixtureEngine(JSON.parse(demo)) - : legacyEngine(dialog.dataset.indexUrl ?? '/docs/search.json'); + : oramaEngine(dialog.dataset.indexUrl ?? '/docs/search-index.json'); let facet = 'all'; let active = -1; @@ -311,6 +311,22 @@ function setup(dialog: HTMLDialogElement) { } }); + // Reaching for the field is the earliest honest signal that someone means to + // search. Once only — `warm` is idempotent, but there is no reason to keep + // asking. + if (engine.warmOn === 'intent') { + let warmed = false; + const onIntent = (event: Event) => { + if (warmed || !opensThis(event.target)) return; + warmed = true; + engine.warm?.(); + }; + // `pointerover` rather than `pointerenter`, which does not bubble to a + // delegated listener, and `focusin` for the keyboard path. + document.addEventListener('pointerover', onIntent); + document.addEventListener('focusin', onIntent); + } + // Chords and shared links belong to the page's own search, not to a demo // instance sitting in an article. if (name !== SITE_SEARCH) return; @@ -328,6 +344,8 @@ function setup(dialog: HTMLDialogElement) { open(); }); + if (engine.warmOn === 'load') engine.warm?.(); + // A shared `?q=` link opens straight into results. const q = new URLSearchParams(window.location.search).get('q'); if (q) open(q); diff --git a/src/scripts/modules/stemmer.js b/src/scripts/modules/stemmer.js deleted file mode 100644 index 778ebcc9c0..0000000000 --- a/src/scripts/modules/stemmer.js +++ /dev/null @@ -1,197 +0,0 @@ -/** - * Removes "morphological and inflexional endings" from words - * See: http://www.tartarus.org/~martin/PorterStemmer - */ -export const stemmer = (function () { - const step2list = { - ational: 'ate', - tional: 'tion', - enci: 'ence', - anci: 'ance', - izer: 'ize', - bli: 'ble', - alli: 'al', - entli: 'ent', - eli: 'e', - ousli: 'ous', - ization: 'ize', - ation: 'ate', - ator: 'ate', - alism: 'al', - iveness: 'ive', - fulness: 'ful', - ousness: 'ous', - aliti: 'al', - iviti: 'ive', - biliti: 'ble', - logi: 'log', - }; - - const step3list = { - icate: 'ic', - ative: '', - alize: 'al', - iciti: 'ic', - ical: 'ic', - ful: '', - ness: '', - }; - - const c = '[^aeiou]', // consonant - v = '[aeiouy]', // vowel - C = c + '[^aeiouy]*', // consonant sequence - V = v + '[aeiou]*', // vowel sequence - mgr0 = '^(' + C + ')?' + V + C, // [C]VC... is m>0 - meq1 = '^(' + C + ')?' + V + C + '(' + V + ')?$', // [C]VC[V] is m=1 - mgr1 = '^(' + C + ')?' + V + C + V + C, // [C]VCVC... is m>1 - s_v = '^(' + C + ')?' + v; // vowel in stem - - /** - * @param {string} w - * @returns {string} - */ - return function (w) { - var stem, - suffix, - firstch, - re, - re2, - re3, - re4, - origword = w; - - if (w.length < 3) { - return w; - } - - firstch = w.substring(0, 1); - - if (firstch == 'y') { - w = firstch.toUpperCase() + w.substring(1, w.length); - } - - // Step 1a - re = /^(.+?)(ss|i)es$/; - re2 = /^(.+?)([^s])s$/; - - if (re.test(w)) { - w = w.replace(re, '$1$2'); - } else if (re2.test(w)) { - w = w.replace(re2, '$1$2'); - } - - // Step 1b - re = /^(.+?)eed$/; - re2 = /^(.+?)(ed|ing)$/; - if (re.test(w)) { - var fp = re.exec(w); - re = new RegExp(mgr0); - if (re.test(fp[1])) { - re = /.$/; - w = w.replace(re, ''); - } - } else if (re2.test(w)) { - var fp = re2.exec(w); - stem = fp[1]; - re2 = new RegExp(s_v); - if (re2.test(stem)) { - w = stem; - re2 = /(at|bl|iz)$/; - re3 = new RegExp('([^aeiouylsz])\\1$'); - re4 = new RegExp('^' + C + v + '[^aeiouwxy]$'); - if (re2.test(w)) { - w = w + 'e'; - } else if (re3.test(w)) { - re = /.$/; - w = w.replace(re, ''); - } else if (re4.test(w)) { - w = w + 'e'; - } - } - } - - // Step 1c - re = /^(.+?)y$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - re = new RegExp(s_v); - if (re.test(stem)) { - w = stem + 'i'; - } - } - - // Step 2 - re = - /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - suffix = fp[2]; - re = new RegExp(mgr0); - if (re.test(stem)) { - w = stem + step2list[suffix]; - } - } - - // Step 3 - re = /^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - suffix = fp[2]; - re = new RegExp(mgr0); - if (re.test(stem)) { - w = stem + step3list[suffix]; - } - } - - // Step 4 - re = - /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/; - re2 = /^(.+?)(s|t)(ion)$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - re = new RegExp(mgr1); - if (re.test(stem)) { - w = stem; - } - } else if (re2.test(w)) { - var fp = re2.exec(w); - stem = fp[1] + fp[2]; - re2 = new RegExp(mgr1); - if (re2.test(stem)) { - w = stem; - } - } - - // Step 5 - re = /^(.+?)e$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - re = new RegExp(mgr1); - re2 = new RegExp(meq1); - re3 = new RegExp('^' + C + v + '[^aeiouwxy]$'); - if (re.test(stem) || (re2.test(stem) && !re3.test(stem))) { - w = stem; - } - } - - re = /ll$/; - re2 = new RegExp(mgr1); - if (re.test(w) && re2.test(w)) { - re = /.$/; - w = w.replace(re, ''); - } - - // and turn initial Y back to y - - if (firstch == 'y') { - w = firstch.toLowerCase() + w.substr(1); - } - - return w; - }; -})(); diff --git a/src/scripts/modules/string.js b/src/scripts/modules/string.js deleted file mode 100644 index dea7142565..0000000000 --- a/src/scripts/modules/string.js +++ /dev/null @@ -1,83 +0,0 @@ -// @ts-check - -/** - * Looks for a search within a string - * - * @param {string} string - * @param {string} search - * @returns - */ -function contains(string, search) { - return string.indexOf(search) > -1; -} - -/** - * Looks for a search within a string - * - * @param {string} string - * @param {string} search - * @returns - */ -function containsWord(string, search) { - return string.split(' ').indexOf(search) > -1; -} - -/** - * - * @param {string} string - * @param {string[]} terms - * @returns - */ -function highlight(string, terms) { - terms.forEach((term) => { - const regEx = new RegExp(term, 'ig'); - const matches = string.match(regEx); - if (matches) { - string = string.replace(regEx, `${matches[0]}`); - } - }); - return string; -} - -/** - * Simplifies a string to plain lower case, removing diacritic characters and hyphens - * This means a search for "co-op" will be found in "COOP" and "Café" will be found in "cafe" - * @param {string} string - * @returns {string} - */ -function sanitise(string) { - // @ts-ignore - if (String.prototype.normalize) { - // Reduces diacritic characters to plain characters - string - .trim() - .normalize('NFD') - .replace(/\./g, ' ') - .replace(/[\u0300-\u036f]/g, '') - .toLowerCase() - .replace(/-/g, ''); - } - - // Some browsers can't normalise strings - return string.trim().toLowerCase().replace(/-/g, ''); -} - -/** - * Sets a minimum length for a search - * @param {string} string - * @returns - */ -function isLongEnough(string) { - return string.length > 1; -} - -/** - * - * @param {string} string - * @returns {string[]} - */ -function explode(string) { - return string.split(' ').filter(isLongEnough).map(sanitise); -} - -export { contains, containsWord, sanitise, explode, highlight }; diff --git a/src/scripts/orama-worker.ts b/src/scripts/orama-worker.ts new file mode 100644 index 0000000000..dd0f6ba9ee --- /dev/null +++ b/src/scripts/orama-worker.ts @@ -0,0 +1,195 @@ +// Holds the Orama index off the main thread. +// +// Restoring a serialized index is CPU-bound and proportional to the corpus, so +// on the main thread it would freeze the page for as long as it takes. The +// overlay is open and taking keystrokes while this runs. +// +// Core `create` + `load` rather than @orama/plugin-data-persistence: the plugin +// reaches for Node's filesystem and buffers, and bundling it for a browser +// worker fails at import time. + +import { create, load, search } from '@orama/orama'; +import { stopwords as englishStopwords } from '@orama/stopwords/english'; + +type LoadMessage = { type: 'load'; indexUrl: string }; +type SearchMessage = { + type: 'search'; + id: number; + query: string; + facet?: string; + limit: number; +}; + +// Has to match the schema the index was built with, or `load` restores an index +// the search side cannot read. +const SCHEMA = { + url: 'string', + title: 'string', + description: 'string', + body: 'string', + trail: 'string', + section: 'enum', +} as const; + +// Has to match `orama-index.ts`, and for a reason that is invisible when it is +// wrong: `load` restores the index data but the tokenizer comes from `create`. +// Build the index with the English stemmer and restore it without, and every +// query term is compared unstemmed against stemmed index terms — `variables` +// found 5 pages instead of 321, and no typo matched at all. Search kept working, +// so nothing failed loudly. +const TOKENIZER = { + stemming: true, + language: 'english', + // Orama defaults this to an empty list, so without it `how`, `do`, `the` and + // 177 others are live search terms on both sides. + stopWords: englishStopwords, +} as const; + +// How much a page's depth counts against it, per path segment. BM25 has no +// notion of a site's shape, so on its own it ranks `/docs/projects/variables` +// below `/docs/projects/variables/system-variables` for the query `variables`: +// the child page repeats the term more often in less text. Readers searching a +// bare section name almost always want the section, and on real search logs this +// single failure was the largest source of missed traffic. +// +// 0.12 was picked by sweeping it against three query sets — the search terms +// readers actually typed, the pages they actually visit, and a curated set. Much +// lower and landing pages stay buried; much higher and a genuinely deep page +// cannot win even when it is the only match. +const DEPTH_PENALTY = 0.12; + +type Ranked = { score: number; document: { url: string; title: string } }; + +/** Lowercased, punctuation collapsed, so `Config as Code` matches `config as code`. */ +function comparable(text: string) { + return text + .toLowerCase() + .replace(/[^a-z0-9 ]+/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +function segments(url: string) { + return url.split('/').filter(Boolean); +} + +/** + * Reorders hits so a section's own page beats the pages inside it. + * + * Two signals, in order. A page the query *names* — by title or by the last + * segment of its URL — wins outright, which is what makes `runbooks` return + * `/docs/runbooks`. Everything else is ordered by score discounted for depth. + * + * A plain tiebreak on depth was tried first and did nothing: BM25 scores are + * floats and almost never tie, so depth has to scale the score rather than break + * a draw between equal ones. + */ +function byNameThenDepth(hits: T[], term: string): T[] { + const wanted = comparable(term); + + const names = (hit: T) => { + const slug = segments(hit.document.url).pop() ?? ''; + return comparable(hit.document.title) === wanted || + comparable(slug.replace(/-/g, ' ')) === wanted + ? 1 + : 0; + }; + + return hits + .map((hit) => ({ + hit, + names: names(hit), + adjusted: + hit.score / (1 + DEPTH_PENALTY * segments(hit.document.url).length), + })) + .sort((a, b) => b.names - a.names || b.adjusted - a.adjusted) + .map((entry) => entry.hit); +} + +let indexUrl = ''; +let ready: Promise | null = null; + +function open() { + ready ??= fetch(indexUrl) + .then((response) => response.json()) + .then((raw) => { + const db = create({ + schema: SCHEMA, + // Matches `orama-index.ts`: the index was built without a sort store, + // so restoring into a database that expects one is a mismatch. + sort: { enabled: false }, + components: { tokenizer: TOKENIZER }, + }); + load(db, raw); + return db; + }) + .catch((error) => { + // A failed load must not poison every later search. + ready = null; + throw error; + }); + + return ready; +} + +self.addEventListener( + 'message', + async (event: MessageEvent) => { + const message = event.data; + + if (message.type === 'load') { + indexUrl = message.indexUrl; + // Kicked off here so the download and restore overlap with typing; the + // rejection is handled inside `open`. + open().catch(() => {}); + return; + } + + try { + const db = await open(); + const query = { + term: message.query, + limit: message.limit, + // Body carries the most text and the least signal per word, so it is + // weighted below the fields that name what a page is about. Title at 8 + // rather than 4 was worth 26 points of Success@5 on real search terms on + // its own — most searches are one or two words that name a page. + boost: { title: 8, description: 2, trail: 2, body: 1 }, + tolerance: 1, + }; + + // Unfiltered, because a `where` clause narrows the facet counts to the + // section being filtered on, and the tab strip has to keep showing what + // the other tabs hold. `count` is the true total rather than the capped + // number of hits returned. + const overview = await search(db as never, { + ...query, + facets: { section: {} }, + }); + + // Filtering in the worker rather than over the returned hits: a section + // whose matches fall outside the first `limit` would otherwise come back + // short. + const selected = + message.facet && message.facet !== 'all' + ? await search(db as never, { + ...query, + where: { section: { eq: message.facet } }, + }) + : overview; + + self.postMessage({ + id: message.id, + counts: { + all: overview.count, + ...(overview.facets?.section?.values ?? {}), + }, + hits: byNameThenDepth(selected.hits, message.query).map( + (hit) => hit.document + ), + }); + } catch (error) { + self.postMessage({ id: message.id, error: String(error) }); + } + } +); diff --git a/src/scripts/search-engine-legacy.ts b/src/scripts/search-engine-legacy.ts deleted file mode 100644 index 398c758757..0000000000 --- a/src/scripts/search-engine-legacy.ts +++ /dev/null @@ -1,265 +0,0 @@ -// The current search, behind the `SearchEngine` seam: one build-time JSON of -// titles, headings, descriptions, tags and an extracted keyword bag, scored in -// the browser. The scoring is carried over unchanged from the results list this -// replaces, so the overlay returns exactly what the old dropdown returned. -// -// Body text is not in the index, which is the ceiling on how good this can be -// and the reason the Pagefind and Orama spikes exist. - -import { contains, explode, highlight, sanitise } from './modules/string.js'; -import { stemmer } from './modules/stemmer.js'; -import { - breadcrumbFrom, - classify, - countByFacet, - type SearchEngine, - type SearchResult, -} from './search-engine'; - -type Heading = { text: string; slug: string; safeText: string }; - -type Entry = { - title: string; - safeTitle: string; - description: string; - keywords: string; - tags: string[]; - headings: Heading[]; - url: string; - depth: number; -}; - -type Scored = Entry & { - score: number; - foundWords: number; - foundTerms: string[]; -}; - -// Phrase and per-term weights, verbatim from the list this replaces. -const SCORING = { - depth: 5, - phraseTitle: 60, - phraseHeading: 20, - phraseDescription: 20, - termTitle: 40, - termHeading: 15, - termDescription: 15, - termTags: 15, - termKeywords: 15, -}; - -const WORD_SCORES = { - titleExact: 20, - titleContains: 15, - headingContains: 10, - contentContains: 1, -}; - -const RESULT_LIMIT = 30; - -async function getSynonyms(): Promise> { - try { - return (await import('./synonyms.js')).synonyms; - } catch { - return {}; - } -} - -/** Terms reach `highlight()` as a regular expression, so a query like `c++` - * would throw before anything was drawn. */ -function escapeForRegExp(term: string) { - return term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); -} - -/** The excerpt is set as HTML so the `` around each hit survives. Escaping - * first means those marks are the only tags that can ever reach the DOM. */ -function escapeHtml(text: string) { - return text - .replace(/&/g, '&') - .replace(//g, '>'); -} - -async function expand(query: string) { - const synonyms = await getSynonyms(); - const terms: string[] = []; - - for (const term of explode(query)) { - const synonym = synonyms[term]; - // An empty synonym is a stop word: drop the term rather than expanding it. - if (synonym === '') continue; - terms.push(term); - if (synonym) terms.push(...synonym.split(' ')); - } - - const stemmed = terms - .map((term) => stemmer(term)) - .filter((stem) => !terms.includes(stem)); - - return { terms, allTerms: [...terms, ...stemmed] }; -} - -function score(entry: Entry, query: string, allTerms: string[]): Scored { - const scored: Scored = { ...entry, score: 0, foundWords: 0, foundTerms: [] }; - - // Phrase matches: the whole query, found intact - if (scored.safeTitle === query) scored.foundWords += WORD_SCORES.titleExact; - - if (contains(scored.safeTitle, query)) { - scored.score += SCORING.phraseTitle; - scored.foundWords += WORD_SCORES.titleContains; - } - - for (const heading of scored.headings) { - if (contains(heading.safeText, query)) { - scored.score += SCORING.phraseHeading; - scored.foundWords += WORD_SCORES.headingContains; - } - } - - if (contains(scored.description, query)) { - scored.score += SCORING.phraseDescription; - scored.foundWords += WORD_SCORES.contentContains; - } - - // Term matches: each word of the query, found anywhere - for (const term of allTerms) { - let found = false; - - if (contains(scored.safeTitle, term)) { - scored.score += SCORING.termTitle; - scored.foundWords += WORD_SCORES.headingContains / 2; - found = true; - } - - for (const heading of scored.headings) { - if (contains(heading.safeText, term)) { - scored.score += SCORING.termHeading; - found = true; - } - } - - if (contains(scored.description, term)) { - scored.score += SCORING.termDescription; - found = true; - } - - for (const tag of scored.tags) { - if (contains(tag, term)) { - scored.score += SCORING.termTags; - found = true; - } - } - - if (contains(scored.keywords, term)) { - scored.score += SCORING.termKeywords; - found = true; - } - - if (found) { - scored.foundWords++; - if (!scored.foundTerms.includes(term)) scored.foundTerms.push(term); - } - } - - // Shallow pages win ties: /docs/features over /docs/features/a/b - if (scored.score > 0) { - if (scored.depth < 5) { - scored.score += SCORING.depth; - scored.foundWords++; - } - if (scored.depth < 4) { - scored.score += SCORING.depth; - scored.foundWords++; - } - } - - return scored; -} - -function byRelevance(a: Scored, b: Scored) { - if (b.foundTerms.length !== a.foundTerms.length) { - return b.foundTerms.length - a.foundTerms.length; - } - if (b.foundWords !== a.foundWords) return b.foundWords - a.foundWords; - return b.score - a.score; -} - -export function legacyEngine(indexUrl: string): SearchEngine { - // One fetch, on the first search rather than on page load, shared by every - // search after it. The old list fetched 1.86MB the moment the script ran. - let loading: Promise | null = null; - - function load() { - loading ??= fetch(indexUrl) - .then((response) => response.json()) - .then((data: Record[]) => - data.map((item) => ({ - title: item.title, - safeTitle: sanitise(item.title), - description: item.description ?? '', - keywords: item.keywords ?? '', - tags: (item.tags ?? []).map((tag: string) => sanitise(tag)), - headings: (item.headings ?? []).map((heading: Heading) => ({ - ...heading, - safeText: sanitise(heading.text), - })), - url: item.url, - depth: item.url.match(/\//g)?.length ?? 0, - })) - ) - .catch(() => { - // A failed load must not poison every later search. - loading = null; - return [] as Entry[]; - }); - - return loading; - } - - return { - warm: load, - - async search(rawQuery, facet) { - // Chained words are joined, so `System.Text` searches as `systemtext`. - const query = sanitise(rawQuery.replace(/\./g, ' ')); - if (!query) return { results: [], counts: countByFacet([]) }; - - const [haystack, { terms, allTerms }] = await Promise.all([ - load(), - expand(query), - ]); - - const highlightTerms = terms.map(escapeForRegExp); - - const matched = haystack - .map((entry) => score(entry, query, allTerms)) - .filter((entry) => entry.score > 0) - .sort(byRelevance) - .slice(0, RESULT_LIMIT) - .map((entry): SearchResult => { - // The index stores absolute production URLs, and every entry in it is - // a page of this site. Keeping only the path is what makes a result - // stay on whatever host is serving it — an ephemeral environment or - // localhost would otherwise send you to production. - const path = new URL(entry.url, window.location.origin).pathname; - - return { - url: path, - title: entry.title, - excerpt: highlight(escapeHtml(entry.description), highlightTerms), - breadcrumb: breadcrumbFrom(path), - ...classify(path), - }; - }); - - return { - counts: countByFacet(matched), - results: - facet && facet !== 'all' - ? matched.filter((result) => result.facet === facet) - : matched, - }; - }, - }; -} diff --git a/src/scripts/search-engine-orama.ts b/src/scripts/search-engine-orama.ts new file mode 100644 index 0000000000..7e52094eac --- /dev/null +++ b/src/scripts/search-engine-orama.ts @@ -0,0 +1,148 @@ +// Orama behind the `SearchEngine` seam. +// +// Unlike Pagefind, the whole index is downloaded and restored into memory before +// the first query can run — there is no per-query fetch at all after that. That +// is the trade the spike exists to measure: a larger up-front cost against zero +// marginal cost, on a corpus this size. +// +// Restore is CPU-bound and would block the main thread, so it happens in a +// worker and queries are messaged across. + +import { + classify, + type SearchEngine, + type SearchResult, +} from './search-engine'; + +const RESULT_LIMIT = 30; + +type WorkerHit = { + url: string; + title: string; + description: string; + body: string; + trail: string; +}; + +type WorkerReply = { + id: number; + hits?: WorkerHit[]; + /** Per-section totals across the whole match set, plus `all`. */ + counts?: Record; + error?: string; +}; + +function escapeHtml(text: string) { + return text + .replace(/&/g, '&') + .replace(//g, '>'); +} + +function escapeForRegExp(term: string) { + return term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +/** + * A window of body text around the first term that matched, with the terms + * marked. Orama returns whole documents rather than excerpts, so the result row + * has to cut its own. + */ +function excerptFrom(hit: WorkerHit, terms: string[]) { + const pattern = new RegExp(terms.map(escapeForRegExp).join('|'), 'i'); + + // Whichever field the query actually hit. Preferring the description outright + // showed every page with a subtitle its subtitle, even when the match that + // earned it its rank was in the body. + // + // `||` rather than `??` on the fallback, and body before description: 1,200 of + // the 1,254 documents have an empty description, and `??` treats "" as a value + // to keep, so it won every time and blocked the fallback to body. + // + // The fallback runs more often than it looks. The index matches stemmed terms + // and this matches literally, so "guided failures" legitimately ranks a page + // whose text only ever says "guides" — the regex then finds nothing. Opening + // the page text is the right answer there; it just arrives unhighlighted. + const matched = [hit.body, hit.description].find( + (text) => text && pattern.test(text) + ); + const source = matched || hit.body || hit.description; + if (!source) return ''; + + const at = source.search(pattern); + const from = at > 60 ? source.lastIndexOf(' ', at - 60) + 1 : 0; + const window = source.slice(from, from + 180); + + const text = escapeHtml((from > 0 ? '…' : '') + window); + return text.replace( + new RegExp(terms.map(escapeForRegExp).join('|'), 'gi'), + (match) => `${match}` + ); +} + +export function oramaEngine(indexUrl: string): SearchEngine { + let worker: Worker | null = null; + let nextId = 0; + const pending = new Map void>(); + + function load() { + if (worker) return; + + worker = new Worker(new URL('./orama-worker.ts', import.meta.url), { + type: 'module', + }); + worker.addEventListener('message', (event: MessageEvent) => { + pending.get(event.data.id)?.(event.data); + pending.delete(event.data.id); + }); + worker.postMessage({ type: 'load', indexUrl }); + } + + function ask(query: string, facet?: string) { + load(); + const id = nextId++; + + return new Promise((resolve) => { + pending.set(id, resolve); + worker!.postMessage({ type: 'search', id, query, facet, limit: 200 }); + }); + } + + return { + // Not on page load: the index is 5.3MB, and warming it for every visitor + // spends that on the majority who never search. On hover the bytes are + // usually already cached, so the 40-67ms parse finishes before the click + // does. + warmOn: 'intent', + warm: load, + + async search(rawQuery, facet) { + const query = rawQuery.trim(); + const empty = { results: [], counts: { all: 0 } }; + if (!query) return empty; + + // The worker applies the facet and reports the counts, because both need + // the whole match set and it is the only side that has it. + const reply = await ask(query, facet); + if (reply.error || !reply.hits || !reply.counts) return empty; + + const terms = query.split(/\s+/).filter((term) => term.length > 1); + + const results = reply.hits.map((hit): SearchResult => { + const path = new URL(hit.url, window.location.origin).pathname; + return { + url: path, + title: hit.title || path, + excerpt: excerptFrom(hit, terms.length > 0 ? terms : [query]), + breadcrumb: hit.trail ? hit.trail.split(' / ') : [], + ...classify(path), + }; + }); + + return { + results: results.slice(0, RESULT_LIMIT), + counts: reply.counts, + }; + }, + }; +} diff --git a/src/scripts/search-engine.ts b/src/scripts/search-engine.ts index c8d1800b2b..feb919f03d 100644 --- a/src/scripts/search-engine.ts +++ b/src/scripts/search-engine.ts @@ -22,6 +22,21 @@ export type SearchEngine = { search(query: string, facet?: string): Promise; /** Optional: start loading the index before the first query needs it. */ warm?(): void; + /** + * When `warm()` is worth running, which depends on what the engine has to + * fetch and parse: + * + * - `load` — on page load. For an engine whose runtime is small enough that + * every visitor can pay for it, searcher or not. + * - `intent` — the first time the pointer or focus reaches a search field. + * Costs nothing for the majority who never go near it, and buys the + * 200-500ms between reaching for the field and clicking it. + * - `open` — not until the overlay opens. The default. + * + * Ctrl/Cmd+K skips straight to `open`, so warming still happens there; this + * only decides how much earlier it can start. + */ + warmOn?: 'load' | 'intent' | 'open'; }; export type Facet = { key: string; label: string }; diff --git a/src/scripts/synonyms.js b/src/scripts/synonyms.js deleted file mode 100644 index 489fc14133..0000000000 --- a/src/scripts/synonyms.js +++ /dev/null @@ -1,17 +0,0 @@ -const synonyms = { - // Keep me alphabetical - licence: 'license', - logs: 'log', - regex: 'regular expression', - // Join words - a: '', - and: '', - for: '', - if: '', - the: '', - to: '', - via: '', - with: '', -}; - -export { synonyms }; diff --git a/tests/api-page.spec.ts b/tests/api-page.spec.ts index acfc11d5f6..e30fa3c53d 100644 --- a/tests/api-page.spec.ts +++ b/tests/api-page.spec.ts @@ -60,14 +60,34 @@ test.describe('api page chrome', () => { }); test('the section is kept out of the search index and the sitemap', async ({ + page, request, }) => { // Temporary, alongside src/lib/underConstruction.ts - delete with it when // the API reference goes live. - const index = await (await request.get('/docs/search.json')).json(); - expect(index.filter((entry) => entry.url.includes('/docs/api/'))).toEqual( - [] + // + // Asked through the overlay rather than of an index file. This used to read + // /docs/search.json, which no longer exists, and the two search engines ship + // indexes of different shapes — one JSON, one a directory of compressed + // chunks. What has to hold is the same either way: a reader searching a word + // the API reference is full of must not be sent into it. + await page.goto('/docs/'); + await page + .locator('input[data-docs-search-trigger]:visible') + .first() + .click(); + await page.locator('[data-docs-search-input]').fill('accounts'); + + // `accounts` is deliberate: it names pages in the API reference *and* pages + // outside it, so the list is never empty. A query that matched nothing would + // pass this test without proving anything. + const results = page.locator('[data-docs-search-results] [role="option"]'); + await expect(results.first()).toBeVisible({ timeout: 30_000 }); + + const hrefs = await results.evaluateAll((rows) => + rows.map((row) => row.getAttribute('href') ?? '') ); + expect(hrefs.filter((href) => href.includes('/docs/api/'))).toEqual([]); const sitemap = await (await request.get('/docs/sitemap.xml')).text(); expect(sitemap).not.toContain('/docs/api/');