diff --git a/package.json b/package.json index 4397f7a2577..088b6c86682 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,8 @@ "quality-check": "npm run typecheck && npm run lint && npm run audit:exports", "quality-check:full": "npm run typecheck && npm run lint && npm run audit:dead-code", "generate:llms": "tsx --require tsconfig-paths/register src/scripts/generate-llms.ts", - "check:llms": "tsx --require tsconfig-paths/register src/scripts/validate-llms.ts" + "check:llms": "tsx --require tsconfig-paths/register src/scripts/validate-llms.ts", + "check:markdown-fidelity": "tsx --require tsconfig-paths/register src/scripts/check-markdown-fidelity.ts" }, "dependencies": { "@11ty/eleventy-fetch": "^4.0.1", diff --git a/src/lib/markdown/__tests__/buildMarkdownArtifact.test.ts b/src/lib/markdown/__tests__/buildMarkdownArtifact.test.ts new file mode 100644 index 00000000000..8312f720ce8 --- /dev/null +++ b/src/lib/markdown/__tests__/buildMarkdownArtifact.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from "@jest/globals" +import { + buildMarkdownArtifact, + normalizeMarkdownPath, + transformPageBodyToMarkdown, +} from "@lib/markdown/buildMarkdownArtifact.js" + +describe("buildMarkdownArtifact", () => { + it.each([ + ["cre/getting-started/cli-installation", "normal"], + ["cre-templates", "special"], + ["cre/reference/sdk/evm-client", "selector"], + ["data-streams/getting-started", "redirect"], + ])("classifies %s as %s", async (requestPath, routeKind) => { + const artifact = await buildMarkdownArtifact(requestPath) + + expect(artifact).not.toBeNull() + expect(artifact?.requestPath).toBe(normalizeMarkdownPath(requestPath)) + expect(artifact?.routeKind).toBe(routeKind) + }) + + it("rejects path escapes", async () => { + expect(normalizeMarkdownPath("../outside")).toBeNull() + await expect(buildMarkdownArtifact("../outside")).resolves.toBeNull() + }) + + it("trims long leading and trailing slash runs", () => { + const slashes = "/".repeat(100_000) + + expect(normalizeMarkdownPath(`${slashes}cre/getting-started${slashes}`)).toBe("cre/getting-started") + }) + + it("accepts an existing extensionless production request path", async () => { + await expect(buildMarkdownArtifact("cre/getting-started/cli-installation")).resolves.toMatchObject({ + requestPath: "cre/getting-started/cli-installation", + routeKind: "normal", + }) + }) + + it.each([".md", ".md.md", ".mdx"])("rejects a leftover %s extension", async (extension) => { + await expect(buildMarkdownArtifact(`cre/getting-started/cli-installation${extension}`)).resolves.toBeNull() + }) + + it.each([ + "cre/getting-started/cli-installation", + "cre/getting-started/cli-installation/macos-linux", + "cre/getting-started/cli-installation/windows", + ])("projects the ordered operating system selector for %s", async (requestPath) => { + const artifact = await buildMarkdownArtifact(requestPath) + const markdown = artifact?.markdown ?? "" + const macosLinux = "[macOS / Linux](/cre/getting-started/cli-installation/macos-linux)" + const windows = "[Windows](/cre/getting-started/cli-installation/windows)" + + expect(markdown).toContain("## Select your operating system") + expect(markdown).toContain(macosLinux) + expect(markdown).toContain(windows) + expect(markdown.indexOf(macosLinux)).toBeLessThan(markdown.indexOf(windows)) + }) +}) + +describe("transformPageBodyToMarkdown", () => { + it("reports the normal transform branch", async () => { + const result = await transformPageBodyToMarkdown("# Kept", "/virtual/normal.mdx") + + expect(result.transformMode).toBe("normal") + expect(result.markdown).toContain("# Kept") + }) + + it("reports the sanitized retry branch", async () => { + const body = `export async function load() { + return @ +} + +# Kept` + const result = await transformPageBodyToMarkdown(body, "/virtual/sanitized.mdx") + + expect(result.transformMode).toBe("sanitized") + expect(result.markdown).toContain("# Kept") + expect(result.markdown).not.toContain("return @") + }) + + it("reports the fallback branch", async () => { + const body = `# Kept + +{` + const result = await transformPageBodyToMarkdown(body, "/virtual/fallback.mdx") + + expect(result).toEqual({ + transformMode: "fallback", + markdown: body, + }) + }) + + it("strips component tags in the fallback branch", async () => { + const result = await transformPageBodyToMarkdown( + `Visible +{`, + "/virtual/fallback-components.mdx" + ) + + expect(result).toEqual({ + transformMode: "fallback", + markdown: `Visible +{`, + }) + }) + + it("preserves a long unterminated repeated component prefix in the fallback branch", async () => { + const body = `${" { + const result = await transformPageBodyToMarkdown("ignored", "/virtual/data-feeds/deprecating-feeds.mdx") + + expect(result.transformMode).toBe("replacement") + expect(result.markdown).toContain("## Deprecated Feeds") + }) +}) diff --git a/src/lib/markdown/__tests__/sourceScanners.test.ts b/src/lib/markdown/__tests__/sourceScanners.test.ts new file mode 100644 index 00000000000..69783689005 --- /dev/null +++ b/src/lib/markdown/__tests__/sourceScanners.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from "@jest/globals" +import { + readStaticDefaultImports, + readStaticJsxSelectorConditions, + removeLeadingMdxFrontmatter, + stripHighlighterComments, +} from "@lib/markdown/sourceScanners.js" + +describe("readStaticDefaultImports", () => { + it("reads single-line and multiline static default imports", () => { + const imports = readStaticDefaultImports(`--- +import Alpha from "./alpha.mdx" +import $Code + from + './code.ts?raw' +import { ignored } from "./named.js" +import "./side-effect.js" +---`) + + expect(Object.fromEntries(imports)).toEqual({ + Alpha: "./alpha.mdx", + $Code: "./code.ts?raw", + }) + }) + + it("scans repeated unterminated import prefixes deterministically", () => { + const source = `${'import Broken from "unterminated\n'.repeat(10_000)}import Kept from "./kept.mdx"` + + expect(Object.fromEntries(readStaticDefaultImports(source))).toEqual({ Kept: "./kept.mdx" }) + expect(Object.fromEntries(readStaticDefaultImports(source))).toEqual({ Kept: "./kept.mdx" }) + }) +}) + +describe("readStaticJsxSelectorConditions", () => { + it("maps static selector values to JSX components", () => { + const conditions = readStaticJsxSelectorConditions( + `{callout === "alpha" && } +{callout + === + 'beta' + && + }`, + "callout" + ) + + expect(Object.fromEntries(conditions)).toEqual({ alpha: "Alpha", beta: "Beta" }) + }) + + it("scans repeated unterminated selector prefixes deterministically", () => { + const source = `${'{callout === "unterminated\n'.repeat(10_000)}{callout === "kept" && }` + + expect(Object.fromEntries(readStaticJsxSelectorConditions(source, "callout"))).toEqual({ kept: "Kept" }) + expect(Object.fromEntries(readStaticJsxSelectorConditions(source, "callout"))).toEqual({ kept: "Kept" }) + }) +}) + +describe("removeLeadingMdxFrontmatter", () => { + it.each([ + ["LF", "---\ntitle: Example\n---\n\n# Body\n", "\n# Body\n"], + ["CRLF", "---\r\ntitle: Example\r\n---\r\n# Body\r\n", "# Body\r\n"], + ["trailing fence whitespace", "--- \ntitle: Example\n--- \n# Body", "# Body"], + ])("removes leading %s frontmatter without changing body newlines", (_name, source, expected) => { + expect(removeLeadingMdxFrontmatter(source)).toBe(expected) + }) + + it("preserves missing and unterminated frontmatter", () => { + expect(removeLeadingMdxFrontmatter("# Body\n---\n")).toBe("# Body\n---\n") + expect(removeLeadingMdxFrontmatter("---\ntitle: Example")).toBe("---\ntitle: Example") + }) +}) + +describe("stripHighlighterComments", () => { + it("removes supported markers while preserving other text and whitespace-only lines", () => { + const code = `const value = 1 // highlight-line + +\t// highlight-start +next // regular comment +end // highlight-end ` + + expect(stripHighlighterComments(code)).toBe(`const value = 1 + + +next // regular comment +end `) + }) +}) diff --git a/src/lib/markdown/__tests__/transformMarkdown.test.ts b/src/lib/markdown/__tests__/transformMarkdown.test.ts index b564bc74200..978900b82ae 100644 --- a/src/lib/markdown/__tests__/transformMarkdown.test.ts +++ b/src/lib/markdown/__tests__/transformMarkdown.test.ts @@ -52,6 +52,218 @@ contract Test { expect(result).toContain("Col1") expect(result).toContain("Col2") }) + + it("projects PageTabs in source order with grouped labels and first URLs", async () => { + const result = await transformMarkdown( + ``, + "/fake/page.mdx" + ) + + expect(result).toContain("## Select your operating system") + expect(result).toContain("[macOS / Linux](/install/macos)") + expect(result).not.toContain("/install/linux") + expect(result.indexOf("[macOS / Linux]")).toBeLessThan(result.indexOf("[Windows]")) + + const withoutHeader = await transformMarkdown( + ``, + "/fake/page.mdx" + ) + expect(withoutHeader).not.toContain("## Guide Versions") + expect(withoutHeader).toContain("[Only](/only)") + }) + + it("pairs Tabs and TabsContent labels with matching panels", async () => { + for (const component of ["Tabs", "TabsContent"]) { + const result = await transformMarkdown( + `<${component}> + First + Second + Second panel + First panel +`, + "/fake/page.mdx" + ) + + expect(result.indexOf("### First")).toBeLessThan(result.indexOf("First panel")) + expect(result.indexOf("First panel")).toBeLessThan(result.indexOf("### Second")) + expect(result.indexOf("### Second")).toBeLessThan(result.indexOf("Second panel")) + } + }) + + it("projects PackageManagerTabs as npm then yarn even when yarn is first", async () => { + const result = await transformMarkdown( + ` + yarn add example + npm install example +`, + "/fake/page.mdx" + ) + + expect(result.indexOf("### npm")).toBeLessThan(result.indexOf("npm install example")) + expect(result.indexOf("npm install example")).toBeLessThan(result.indexOf("### yarn")) + expect(result.indexOf("### yarn")).toBeLessThan(result.indexOf("yarn add example")) + }) + + it("unwraps Fragment content", async () => { + const result = await transformMarkdown(`Visible **content**`, "/fake/page.mdx") + expect(result).toContain("Visible **content**") + expect(result).not.toContain("Fragment") + }) + + it("projects Accordion number, title, and body", async () => { + const result = await transformMarkdown( + ` +Body instructions. +`, + "/fake/page.mdx" + ) + expect(result).toContain("### 2. Deploy the contract") + expect(result).toContain("Body instructions.") + }) + + it("uses an Accordion title slot instead of the title prop", async () => { + const withTitleSlot = await transformMarkdown( + ` + Review the deployment +Body remains visible. +`, + "/fake/page.mdx" + ) + expect(withTitleSlot).toContain("### 3. Review the deployment") + expect(withTitleSlot).not.toContain("Ignored prop title") + expect(withTitleSlot).toContain("Body remains visible.") + }) + + it("projects Address with exact URLs and static truncation", async () => { + const result = await transformMarkdown( + `Exact:
+Truncated:
`, + "/fake/page.mdx" + ) + expect(result).toContain("[0x1234567890abcdef](https://example.test/address/0x1234567890abcdef?view=code)") + expect(result).toContain("[0x1234...cdef](https://example.test/exact)") + }) + + it("projects a block ClickToZoom as an exact Markdown image", async () => { + const result = await transformMarkdown( + ``, + "/fake/page.mdx" + ) + + expect(result).toBe("![Architecture diagram](/images/architecture.png)\n") + }) + + it("projects an inline ClickToZoom with default alt text", async () => { + const result = await transformMarkdown(`Before after.`, "/fake/page.mdx") + + expect(result).toBe("Before ![Image](/images/detail.png) after.\n") + }) + + it("projects ClickToZoom through the AST with a long repeated attribute value", async () => { + const repeated = " =".repeat(50_000) + const result = await transformMarkdown( + ``, + "/fake/page.mdx" + ) + + expect(result).toBe("![Detail](/images/detail.png)\n") + }) + + it("projects Aside as a Markdown blockquote", async () => { + const result = await transformMarkdown( + ``, + "/fake/page.mdx" + ) + expect(result).toContain("> **WARNING: Important**") + expect(result).toContain("> Read the warning.") + }) + + it("projects an Aside with a long repeated attribute value", async () => { + const repeated = "a".repeat(100_000) + const result = await transformMarkdown( + ``, + "/fake/page.mdx" + ) + expect(result).toContain(`> **NOTE: ${repeated}**`) + expect(result).toContain("> Body.") + }) + + it("projects Callout like Aside", async () => { + const result = await transformMarkdown( + ` +Read the warning. +`, + "/fake/page.mdx" + ) + expect(result).toContain("> **CAUTION: Check this**") + expect(result).toContain("> Read the warning.") + }) + + it("projects SchemaFieldsTable from report schema definitions", async () => { + const result = await transformMarkdown(``, "/fake/page.mdx") + expect(result).toContain("| Field") + expect(result).toContain("`feedId`") + expect(result).toContain("`price`") + expect(result).toContain("Time-weighted average price") + }) + + it("projects every static CodeHighlightBlockMulti language when no target is set", async () => { + const result = await transformMarkdown( + ``, + "/fake/page.mdx" + ) + expect(result).toContain("```ts") + expect(result).toContain("const answer = 42") + expect(result).toContain("```go") + expect(result).toContain("package main") + expect(result.indexOf("```ts")).toBeLessThan(result.indexOf("```go")) + + const selected = await transformMarkdown( + ``, + "/fake/page.mdx", + { targetLanguage: "go" } + ) + expect(selected).not.toContain("```ts") + expect(selected).not.toContain("ts only") + expect(selected).toContain("```go") + expect(selected).toContain("go only") + }) + + it("removes residual MDX, HTML, ESM, and nonliteral projections", async () => { + const result = await transformMarkdown( + `import Unknown from "./Unknown" + +Before hidden JSX after. +hidden HTML +{dynamicValue} +`, + "/fake/page.mdx" + ) + expect(result).toContain("Before") + expect(result).toContain("after.") + expect(result).not.toMatch(/<[/A-Za-z]/) + expect(result).not.toContain("import Unknown") + expect(result).not.toContain("{dynamicValue}") + expect(result).not.toContain("dynamicPages") + }) }) describe("extractFrontmatter", () => { diff --git a/src/lib/markdown/buildMarkdownArtifact.ts b/src/lib/markdown/buildMarkdownArtifact.ts new file mode 100644 index 00000000000..4c466d18dc8 --- /dev/null +++ b/src/lib/markdown/buildMarkdownArtifact.ts @@ -0,0 +1,378 @@ +import fs from "node:fs/promises" +import path from "node:path" +import { transformPageToMarkdown } from "./transformMarkdown.js" +import type { MarkdownArtifact } from "./types.js" +import { extractFrontmatter, getIsoStringOrUndefined, toCanonicalUrl, toContentRelative } from "./utils.js" + +const SITE_BASE = "https://docs.chain.link" +const CONTENT_ROOT = path.resolve("src/content") +const LLMS_DIRECTIVE = "> For the complete documentation index, see [llms.txt](/llms.txt)." + +const MARKDOWN_REDIRECTS: Record = { + "ccip/tutorials/cross-chain-tokens": "ccip/tutorials/evm/cross-chain-tokens", + + // Data Streams + "data-streams/getting-started": "data-streams/tutorials/streams-trade/getting-started", + "data-streams/getting-started-hardhat": "data-streams/tutorials/streams-trade/getting-started-hardhat", + "data-streams/reference/streams-direct/streams-direct-onchain-verification": + "data-streams/reference/onchain-verification", + + // Newly surfaced redirects + "chainlink-functions/resources/concepts": "chainlink-functions/resources", + "cre/getting-started/conclusion": "cre/getting-started", + "data-streams/reference/streams-direct/streams-direct-interface-ws": "data-streams/reference/interface-ws", +} + +type TransformOutcome = Pick + +type SpecialResolution = { + resolvedPath: string + sourceCanonicalPath: string + sourcePath: string +} + +type CreResolution = + | { kind: "none" } + | { kind: "resolved"; path: string; sourcePath: string } + | { kind: "selector"; goPath: string; tsPath: string } + +export function normalizeMarkdownPath(pathParam: string | undefined): string | null { + if (!pathParam) return null + + let start = 0 + let end = pathParam.length + while (start < end && pathParam[start] === "/") start++ + while (end > start && pathParam[end - 1] === "/") end-- + const cleanPath = pathParam.slice(start, end) + + if (!cleanPath || /\.(?:md|mdx)$/i.test(cleanPath)) return null + + const segments = cleanPath.split("/") + if (segments.some((segment) => segment === ".." || segment === "." || segment === "")) { + return null + } + + return cleanPath +} + +export async function buildMarkdownArtifact( + requestPath: string, + options: { lang?: string } = {} +): Promise { + const cleanPath = normalizeMarkdownPath(requestPath) + if (!cleanPath) return null + + const specialResolution = await resolveSpecialCanonicalMarkdownPath(cleanPath) + if (specialResolution) { + return buildMarkdownArtifactFromPath( + cleanPath, + specialResolution.resolvedPath, + "special", + options, + specialResolution.sourcePath, + specialResolution.sourceCanonicalPath + ) + } + + const creResolution = await resolveCreCanonicalMarkdownPath(cleanPath) + if (creResolution.kind === "selector") { + return { + requestPath: cleanPath, + routeKind: "selector", + transformMode: "normal", + markdown: buildCreSelectorMarkdown(cleanPath, creResolution), + } + } + + const resolvedPath = creResolution.kind === "resolved" ? creResolution.path : cleanPath + const redirectTarget = MARKDOWN_REDIRECTS[resolvedPath] + if (redirectTarget) { + return { + requestPath: cleanPath, + routeKind: "redirect", + transformMode: "normal", + markdown: buildMarkdownMovedBody(resolvedPath, redirectTarget), + } + } + + return buildMarkdownArtifactFromPath( + cleanPath, + resolvedPath, + "normal", + options, + creResolution.kind === "resolved" ? creResolution.sourcePath : undefined + ) +} + +export async function transformPageBodyToMarkdown( + body: string, + mdxAbsPath: string, + options: { siteBase?: string; targetLanguage?: string } = {} +): Promise { + if (mdxAbsPath.includes("data-feeds/deprecating-feeds")) { + return { + transformMode: "replacement", + markdown: ` +## Deprecated Feeds + +This page contains dynamically generated or component-heavy content. + +For the full and most up-to-date information, see: +https://docs.chain.link/data-feeds/deprecating-feeds +`.trim(), + } + } + + const transformOptions = { + siteBase: options.siteBase ?? SITE_BASE, + targetLanguage: options.targetLanguage, + } + + try { + return { + transformMode: "normal", + markdown: await transformPageToMarkdown(body, mdxAbsPath, transformOptions), + } + } catch { + const sanitizedBody = stripRuntimeMdxSyntax(body) + + try { + return { + transformMode: "sanitized", + markdown: await transformPageToMarkdown(sanitizedBody, mdxAbsPath, transformOptions), + } + } catch { + return { + transformMode: "fallback", + markdown: buildFallbackMarkdownBody(sanitizedBody), + } + } + } +} + +async function resolveSpecialCanonicalMarkdownPath(cleanPath: string): Promise { + const specialPathMap: Record = { + "cre-templates": "cre/templates", + } + + const resolvedPath = specialPathMap[cleanPath] + if (!resolvedPath) return null + + const sourcePath = await findContentFile(resolvedPath) + if (!sourcePath) return null + + return { + resolvedPath, + sourceCanonicalPath: cleanPath, + sourcePath, + } +} + +async function resolveCreCanonicalMarkdownPath(cleanPath: string): Promise { + if (!cleanPath.startsWith("cre/")) { + return { kind: "none" } + } + + const direct = await findContentFile(cleanPath) + if (direct) { + return { kind: "resolved", path: cleanPath, sourcePath: direct } + } + + const goPath = `${cleanPath}-go` + const tsPath = `${cleanPath}-ts` + const [goFile, tsFile] = await Promise.all([findContentFile(goPath), findContentFile(tsPath)]) + + if (goFile && tsFile) { + return { kind: "selector", goPath, tsPath } + } + + if (goFile) { + return { kind: "resolved", path: goPath, sourcePath: goFile } + } + + if (tsFile) { + return { kind: "resolved", path: tsPath, sourcePath: tsFile } + } + + return { kind: "none" } +} + +async function buildMarkdownArtifactFromPath( + requestPath: string, + resolvedPath: string, + routeKind: "normal" | "special", + options: { lang?: string }, + knownSourcePath?: string, + sourceCanonicalPathOverride?: string +): Promise { + const sourcePath = knownSourcePath ?? (await findContentFile(resolvedPath)) + if (!sourcePath) return null + + const raw = await fs.readFile(sourcePath, "utf-8") + const { body, fmTitle, fmLastModified } = extractFrontmatter(raw) + const transformed = await transformPageBodyToMarkdown(body, sourcePath, { + siteBase: SITE_BASE, + targetLanguage: options.lang, + }) + + const section = resolvedPath.split("/")[0] + const relFromContent = toContentRelative(sourcePath) + const derivedSourceUrl = toCanonicalUrl(section, relFromContent, SITE_BASE) + const sourceUrl = sourceCanonicalPathOverride ? `${SITE_BASE}/${sourceCanonicalPathOverride}` : derivedSourceUrl + const title = fmTitle || path.basename(sourcePath, path.extname(sourcePath)) + const lastModified = getIsoStringOrUndefined(fmLastModified) + const headerLines = [ + `# ${title}`, + `Source: ${sourceUrl}`, + ...(lastModified ? [`Last Updated: ${lastModified}`] : []), + "", + LLMS_DIRECTIVE, + "", + ] + + return { + requestPath, + routeKind, + transformMode: transformed.transformMode, + sourcePath: relFromContent, + markdown: [...headerLines, transformed.markdown.trim()].join("\n"), + } +} + +async function findContentFile(cleanPath: string): Promise { + const possiblePaths = [ + path.resolve(CONTENT_ROOT, `${cleanPath}.mdx`), + path.resolve(CONTENT_ROOT, cleanPath, "index.mdx"), + path.resolve(CONTENT_ROOT, `${cleanPath}.md`), + path.resolve(CONTENT_ROOT, cleanPath, "index.md"), + ] + + for (const candidate of possiblePaths) { + if (!candidate.startsWith(`${CONTENT_ROOT}${path.sep}`)) continue + try { + await fs.access(candidate) + return candidate + } catch {} + } + + return null +} + +function buildFallbackMarkdownBody(body: string): string { + return stripFallbackComponentTags(stripRuntimeMdxSyntax(body)).trim() +} + +function stripFallbackComponentTags(body: string): string { + const chunks: string[] = [] + let copiedThrough = 0 + let searchFrom = 0 + + while (searchFrom < body.length) { + const tagStart = body.indexOf("<", searchFrom) + if (tagStart === -1) break + + const nameStart = body.charCodeAt(tagStart + 1) === 47 ? tagStart + 2 : tagStart + 1 + const firstNameChar = body.charCodeAt(nameStart) + if (firstNameChar < 65 || firstNameChar > 90) { + searchFrom = tagStart + 1 + continue + } + + const tagEnd = body.indexOf(">", nameStart + 1) + if (tagEnd === -1) { + chunks.push(body.slice(copiedThrough)) + return chunks.join("") + } + + chunks.push(body.slice(copiedThrough, tagStart)) + copiedThrough = tagEnd + 1 + searchFrom = copiedThrough + } + + chunks.push(body.slice(copiedThrough)) + return chunks.join("") +} + +function stripRuntimeMdxSyntax(body: string): string { + const lines = body.split("\n") + const output: string[] = [] + let skippingExportBlock = false + let skippingImportBlock = false + let braceDepth = 0 + + for (const line of lines) { + const trimmed = line.trim() + + if (skippingImportBlock) { + if (trimmed.includes(" from ") || trimmed.endsWith('"') || trimmed.endsWith("'")) { + skippingImportBlock = false + } + continue + } + + if (skippingExportBlock) { + braceDepth += countChar(line, "{") + braceDepth -= countChar(line, "}") + + if (braceDepth <= 0) { + skippingExportBlock = false + braceDepth = 0 + } + continue + } + + if (/^import\s+/.test(trimmed)) { + if (!trimmed.includes(" from ")) skippingImportBlock = true + continue + } + + if (/^export\s+(async\s+)?function\s+/.test(trimmed)) { + skippingExportBlock = true + braceDepth = countChar(line, "{") - countChar(line, "}") + continue + } + + if (/^export\s+(const|let|var)\s+/.test(trimmed)) { + continue + } + + output.push(line) + } + + return output.join("\n") +} + +function countChar(value: string, char: string): number { + return value.split(char).length - 1 +} + +function buildMarkdownMovedBody(sourcePath: string, targetPath: string): string { + const sourceUrl = `${SITE_BASE}/${sourcePath}` + const targetUrl = `/${targetPath}.md` + + return [ + "# Redirect", + `Source: ${sourceUrl}`, + "", + LLMS_DIRECTIVE, + "", + "This page has moved.", + "", + `Use the current documentation: [${targetPath}](${targetUrl}).`, + "", + ].join("\n") +} + +function buildCreSelectorMarkdown(canonicalPath: string, resolution: { goPath: string; tsPath: string }): string { + const canonicalUrl = `${SITE_BASE}/${canonicalPath}` + return [ + `# ${canonicalPath}`, + `Source: ${canonicalUrl}`, + "", + LLMS_DIRECTIVE, + "", + `- Go: /${resolution.goPath}.md`, + `- TypeScript: /${resolution.tsPath}.md`, + "", + ].join("\n") +} diff --git a/src/lib/markdown/componentHandlers.ts b/src/lib/markdown/componentHandlers.ts index 9db02c21a32..84e836f0143 100644 --- a/src/lib/markdown/componentHandlers.ts +++ b/src/lib/markdown/componentHandlers.ts @@ -6,11 +6,163 @@ import fs from "fs" import path from "path" import type { Parent, Literal, Node } from "unist" import type { MdxJsxNode, ComponentContext } from "./types.js" +import { + readStaticDefaultImports, + readStaticJsxSelectorConditions, + removeLeadingMdxFrontmatter, + stripHighlighterComments, +} from "./sourceScanners.js" import { calculateNetworkFeesForTokenMechanismDirect, calculateMessagingNetworkFeesDirect, - TokenMechanism, -} from "../../config/data/ccip/index.js" +} from "../../config/data/ccip/utils.js" +import { TokenMechanism } from "../../config/data/ccip/types.js" +import { REPORT_SCHEMA_DEFINITIONS } from "../../features/feeds/components/reportSchemaData.js" + +type StaticValue = null | boolean | number | string | StaticValue[] | { [key: string]: StaticValue } +type EstreeNode = { + type?: string + value?: unknown + name?: string + operator?: string + argument?: EstreeNode + elements?: (EstreeNode | null)[] + properties?: EstreeNode[] + key?: EstreeNode + computed?: boolean + kind?: string + method?: boolean + shorthand?: boolean + expressions?: EstreeNode[] + quasis?: { value?: { cooked?: string | null; raw?: string } }[] +} + +const NON_STATIC = Symbol("non-static") + +function staticEstreeValue(node: EstreeNode | undefined): StaticValue | typeof NON_STATIC { + if (!node) return NON_STATIC + if (node.type === "Literal") { + return node.value === null || ["boolean", "number", "string"].includes(typeof node.value) + ? (node.value as StaticValue) + : NON_STATIC + } + if (node.type === "TemplateLiteral" && node.expressions?.length === 0 && node.quasis?.length === 1) { + return node.quasis[0].value?.cooked ?? node.quasis[0].value?.raw ?? "" + } + if (node.type === "UnaryExpression" && (node.operator === "+" || node.operator === "-")) { + const value = staticEstreeValue(node.argument) + return typeof value === "number" ? (node.operator === "-" ? -value : value) : NON_STATIC + } + if (node.type === "ArrayExpression") { + const values: StaticValue[] = [] + for (const element of node.elements || []) { + if (!element) return NON_STATIC + const value = staticEstreeValue(element) + if (value === NON_STATIC) return NON_STATIC + values.push(value) + } + return values + } + if (node.type === "ObjectExpression") { + const value: { [key: string]: StaticValue } = {} + for (const property of node.properties || []) { + if ( + property.type !== "Property" || + property.computed || + property.kind !== "init" || + property.method || + property.shorthand + ) { + return NON_STATIC + } + const key = + property.key?.type === "Identifier" + ? property.key.name + : property.key?.type === "Literal" && + (typeof property.key.value === "string" || typeof property.key.value === "number") + ? String(property.key.value) + : undefined + const propertyValue = staticEstreeValue(property.value as EstreeNode) + if (key === undefined || propertyValue === NON_STATIC) return NON_STATIC + value[key] = propertyValue + } + return value + } + return NON_STATIC +} + +function staticAttribute(node: MdxJsxNode, name: string): StaticValue | typeof NON_STATIC | undefined { + const attribute = node.attributes?.find((candidate) => candidate.name === name) + if (!attribute) return undefined + const rawValue = attribute.value as unknown + if (rawValue === null || rawValue === undefined) return true + if (typeof rawValue === "string") return rawValue + if (typeof rawValue !== "object") return NON_STATIC + const expression = ( + rawValue as { + data?: { estree?: { body?: { expression?: EstreeNode }[] } } + } + ).data?.estree?.body?.[0]?.expression + return staticEstreeValue(expression) +} + +function dropNode(parent: Parent, index: number): number { + parent.children.splice(index, 1) + return index +} + +function textNode(value: string): Literal { + return { type: "text", value } as Literal +} + +function headingNode(depth: number, value: string): Parent { + return { type: "heading", depth, children: [textNode(value)] } as Parent +} + +function paragraphNode(children: Node[]): Parent { + return { type: "paragraph", children } as Parent +} + +function linkNode(label: string, url: string): Parent { + return { type: "link", url, children: [textNode(label)] } as Parent +} + +function staticNodeText(node: Node): string | typeof NON_STATIC { + if (node.type === "text" || node.type === "inlineCode") { + return typeof (node as Literal).value === "string" ? String((node as Literal).value) : NON_STATIC + } + if (node.type === "break") return " " + if (node.type === "paragraph" || node.type === "emphasis" || node.type === "strong" || node.type === "delete") { + const parts: string[] = [] + for (const child of (node as Parent).children || []) { + const part = staticNodeText(child) + if (part === NON_STATIC) return NON_STATIC + parts.push(part) + } + return parts.join("") + } + return NON_STATIC +} + +function staticChildrenText(node: Parent): string | typeof NON_STATIC { + const parts: string[] = [] + for (const child of node.children || []) { + const part = staticNodeText(child) + if (part === NON_STATIC) return NON_STATIC + parts.push(part) + } + return parts.join("").trim() +} + +function resolveExistingWithin(root: string, candidate: string): string | undefined { + try { + const realRoot = fs.realpathSync(root) + const realCandidate = fs.realpathSync(path.resolve(root, candidate)) + if (realCandidate === realRoot || realCandidate.startsWith(realRoot + path.sep)) return realCandidate + } catch { + // Missing files are not projectable. + } +} /** * Load CcipCommon callout mapping dynamically from CcipCommon.astro @@ -22,24 +174,16 @@ export function loadCcipCommonMapping(): Record { const astroContent = fs.readFileSync(astroFilePath, "utf-8") // First, build a map of Component names to file paths from imports - const importRegex = /import\s+(\w+)\s+from\s+["'](.+?)["']/g - const componentToFile: Record = {} - - for (const match of astroContent.matchAll(importRegex)) { - const [, componentName, filePath] = match - const cleanPath = filePath.replace(/^\.\//, "") - componentToFile[componentName] = cleanPath - } + const componentToFile = readStaticDefaultImports(astroContent) // Then, parse the conditional statements to map callout names to component names - const conditionalRegex = /callout\s+===\s+["'](\w+)["']\s+&&\s+<(\w+)/g + const conditions = readStaticJsxSelectorConditions(astroContent, "callout") const mapping: Record = {} - for (const match of astroContent.matchAll(conditionalRegex)) { - const [, calloutName, componentName] = match - const filePath = componentToFile[componentName] + for (const [calloutName, componentName] of conditions) { + const filePath = componentToFile.get(componentName) if (filePath) { - mapping[calloutName] = filePath + mapping[calloutName] = filePath.startsWith("./") ? filePath.slice(2) : filePath } } @@ -74,15 +218,13 @@ export function handleCcipCommon( const fileName = calloutFileMap[calloutValue] if (fileName) { - const calloutPath = path.resolve("src/features/ccip", fileName) + const calloutPath = resolveExistingWithin(path.resolve("src/features/ccip"), fileName) - if (fs.existsSync(calloutPath)) { + if (calloutPath) { let calloutContent = fs.readFileSync(calloutPath, "utf-8") // Strip frontmatter if present - if (calloutContent.trim().startsWith("---")) { - calloutContent = calloutContent.replace(/^---\s*\n[\s\S]*?\n---\s*\n/, "") - } + calloutContent = removeLeadingMdxFrontmatter(calloutContent) // Strip import statements calloutContent = calloutContent.replace(/^import\s+.+$/gm, "").trim() @@ -91,13 +233,15 @@ export function handleCcipCommon( const calloutTree = context.processor.parse(calloutContent) if (calloutTree && calloutTree.children) { parent.children.splice(index, 1, ...calloutTree.children) - return index + calloutTree.children.length + return index } } } } + return dropNode(parent, index) } catch (e) { console.warn(`Failed to process CcipCommon in ${context.mdxAbsPath}:`, e) + return dropNode(parent, index) } } @@ -121,19 +265,17 @@ export function handleCodeHighlightBlock( ?.data?.estree?.body?.[0]?.expression?.name if (codeVarName) { - const importRegex = new RegExp(`import\\s+${codeVarName}\\s+from\\s+['"](.+?)['"]`) - const match = context.markdown.match(importRegex) - - if (match) { - const importPath = match[1].split("?")[0] // Strip "?raw" and other query params - const codeAbsPath = path.resolve(path.dirname(context.mdxAbsPath), importPath) - let codeContent = fs.readFileSync(codeAbsPath, "utf-8") - - // Strip highlighter comments - codeContent = codeContent - .split("\n") - .map((line) => line.replace(/\s*\/\/\s*highlight-(line|start|end)/, "")) - .join("\n") + const importPath = readStaticDefaultImports(context.markdown).get(codeVarName)?.split("?")[0] + + if (importPath) { + const codeAbsPath = resolveExistingWithin( + process.cwd(), + path.resolve(path.dirname(context.mdxAbsPath), importPath) + ) + if (!codeAbsPath) { + return dropNode(parent, index) + } + const codeContent = stripHighlighterComments(fs.readFileSync(codeAbsPath, "utf-8")) const langAttr = node.attributes?.find((a) => a.name === "lang") const titleAttr = node.attributes?.find((a) => a.name === "title") @@ -155,8 +297,10 @@ export function handleCodeHighlightBlock( return index + newNodes.length } } + return dropNode(parent, index) } catch (e) { console.warn(`Failed to process CodeHighlightBlock in ${context.mdxAbsPath}:`, e) + return dropNode(parent, index) } } @@ -175,72 +319,75 @@ export function handleCodeHighlightBlockMulti( context: ComponentContext ): number | void { try { - const languagesAttr = node.attributes?.find((a) => a.name === "languages") - - if (languagesAttr && context.targetLanguage) { - // Extract the code variable name for the target language - // The structure is: languages={{ go: { code: goVar }, ts: { code: tsVar } }} - const attrValue = languagesAttr.value - const estreeBody = - typeof attrValue === "object" && attrValue && "data" in attrValue - ? attrValue.data?.estree?.body?.[0] - : undefined - const languagesObj = - estreeBody && typeof estreeBody === "object" && "expression" in estreeBody - ? (estreeBody.expression as { properties?: unknown })?.properties - : undefined - - if (languagesObj) { - for (const langProp of languagesObj as Record[]) { - const langKey = - (langProp.key as { name?: string; value?: string })?.name || - (langProp.key as { name?: string; value?: string })?.value - - if (langKey === context.targetLanguage) { - const codeProperty = (langProp.value as { properties?: Record[] })?.properties?.find( - (p) => (p.key as { name?: string })?.name === "code" - ) - const codeVarName = (codeProperty?.value as { name?: string })?.name - - if (codeVarName) { - // Find the import statement for this variable - const importRegex = new RegExp(`import\\s+${codeVarName}\\s+from\\s+['"](.+?)['"]`) - const match = context.markdown.match(importRegex) - - if (match) { - const importPath = match[1].split("?")[0] // Strip "?raw" - const codeAbsPath = path.resolve(path.dirname(context.mdxAbsPath), importPath) - let codeContent = fs.readFileSync(codeAbsPath, "utf-8") - - // Strip highlighter comments - codeContent = codeContent - .split("\n") - .map((line) => line.replace(/\s*\/\/\s*highlight-(line|start|end)/, "")) - .join("\n") - - // Infer language from file extension - const fileExt = path.extname(codeAbsPath).slice(1) - const lang = fileExt || context.targetLanguage - - // Create a code block for this language - const newNodes: Node[] = [] - newNodes.push({ - type: "code", - lang, - value: codeContent.trim(), - } as Literal) - - parent.children.splice(index, 1, ...newNodes) - return index + newNodes.length - } - } - break + const languagesAttr = node.attributes?.find((attribute) => attribute.name === "languages") + const expression = ( + languagesAttr?.value as { + data?: { estree?: { body?: { expression?: EstreeNode }[] } } + } + )?.data?.estree?.body?.[0]?.expression + if (expression?.type !== "ObjectExpression") { + return dropNode(parent, index) + } + + const codeNodes: Node[] = [] + const imports = readStaticDefaultImports(context.markdown) + for (const languageProperty of expression.properties || []) { + if (languageProperty.type !== "Property" || languageProperty.computed) continue + const language = + languageProperty.key?.type === "Identifier" + ? languageProperty.key.name + : languageProperty.key?.type === "Literal" && typeof languageProperty.key.value === "string" + ? languageProperty.key.value + : undefined + if (!language || (context.targetLanguage && language !== context.targetLanguage)) continue + + const languageConfig = languageProperty.value as EstreeNode + if (languageConfig?.type !== "ObjectExpression") continue + const codeProperty = (languageConfig.properties || []).find((property) => { + if (property.type !== "Property" || property.computed) return false + return ( + (property.key?.type === "Identifier" && property.key.name === "code") || + (property.key?.type === "Literal" && property.key.value === "code") + ) + }) + const codeExpression = codeProperty?.value as EstreeNode | undefined + let code: string | undefined + let codeLanguage = language + + if (codeExpression?.type === "Identifier" && codeExpression.name) { + const importPath = imports.get(codeExpression.name)?.split("?")[0] + if (importPath) { + const codePath = resolveExistingWithin( + process.cwd(), + path.resolve(path.dirname(context.mdxAbsPath), importPath) + ) + if (codePath) { + code = fs.readFileSync(codePath, "utf-8") + codeLanguage = path.extname(codePath).slice(1) || language } } + } else { + const staticCode = staticEstreeValue(codeExpression) + if (typeof staticCode === "string") code = staticCode } + + if (code !== undefined) { + codeNodes.push({ + type: "code", + lang: codeLanguage, + value: stripHighlighterComments(code).trim(), + } as Literal) + } + } + + if (codeNodes.length === 0) { + return dropNode(parent, index) } + parent.children.splice(index, 1, ...codeNodes) + return index + codeNodes.length } catch (e) { console.warn(`Failed to process CodeHighlightBlockMulti in ${context.mdxAbsPath}:`, e) + return dropNode(parent, index) } } @@ -294,45 +441,31 @@ export function handleDiv(node: MdxJsxNode, parent: Parent, index: number): numb */ export function handleAside(node: MdxJsxNode, parent: Parent, index: number, context: ComponentContext): number | void { try { - const typeAttr = node.attributes?.find((a) => a.name === "type") - const titleAttr = node.attributes?.find((a) => a.name === "title") - - const type = typeof typeAttr?.value === "string" ? typeAttr.value.toUpperCase() : "NOTE" - const title = typeof titleAttr?.value === "string" ? titleAttr.value : "" - - // Get children content - const children = (node as Parent).children || [] - - if (children.length === 0) { - return + const typeValue = staticAttribute(node, "type") + const titleValue = staticAttribute(node, "title") + if ( + typeValue === NON_STATIC || + (typeValue !== undefined && typeof typeValue !== "string") || + titleValue === NON_STATIC || + (titleValue !== undefined && typeof titleValue !== "string") + ) { + return dropNode(parent, index) } - - // Create blockquote header - const header = title ? `**${type}: ${title}**` : `**${type}**` - - // Create new nodes for blockquote - const newNodes: Node[] = [] - - // Add blockquote paragraph with header - newNodes.push({ + const type = typeof typeValue === "string" ? typeValue.toUpperCase() : "NOTE" + const title = typeof titleValue === "string" ? titleValue : "" + const header = title ? `${type}: ${title}` : type + const blockquote = { type: "blockquote", children: [ - { - type: "paragraph", - children: [{ type: "text", value: header } as Literal], - } as Parent, - { - type: "paragraph", - children: [{ type: "text", value: "" } as Literal], - } as Parent, - ...children, + paragraphNode([{ type: "strong", children: [textNode(header)] } as Parent]), + ...((node as Parent).children || []), ], - } as Parent) - - parent.children.splice(index, 1, ...newNodes) - return index + newNodes.length + } as Parent + parent.children.splice(index, 1, blockquote) + return index } catch (e) { console.warn(`Failed to process Aside in ${context.mdxAbsPath}:`, e) + return dropNode(parent, index) } } @@ -428,9 +561,10 @@ export function handleCodeSample( const possiblePaths = [publicPath, path.resolve(src), path.join(process.cwd(), "src", src)] let codeContent: string | null = null - for (const p of possiblePaths) { - if (fs.existsSync(p)) { - codeContent = fs.readFileSync(p, "utf-8") + for (const candidate of possiblePaths) { + const safePath = resolveExistingWithin(process.cwd(), candidate) + if (safePath) { + codeContent = fs.readFileSync(safePath, "utf-8") break } } @@ -466,18 +600,12 @@ export function handleCodeSample( /** * Handle Billing component - generate markdown table with CCIP network fees - * @param node - AST node * @param parent - Parent node * @param index - Index in parent's children * @param context - Component context * @returns New index or void */ -export function handleBilling( - node: MdxJsxNode, - parent: Parent, - index: number, - context: ComponentContext -): number | void { +export function handleBilling(parent: Parent, index: number, context: ComponentContext): number | void { try { // Calculate fees using the same logic as Billing.astro const lockAndUnlockAllLanes = calculateNetworkFeesForTokenMechanismDirect(TokenMechanism.LockAndUnlock, "allLanes") @@ -545,3 +673,289 @@ export function handleBilling( } as Parent } } + +export function handlePageTabs(node: MdxJsxNode, parent: Parent, index: number): number | void { + const pages = staticAttribute(node, "pages") + const showHeader = staticAttribute(node, "showHeader") + const headerTitle = staticAttribute(node, "headerTitle") + if ( + pages === NON_STATIC || + !Array.isArray(pages) || + showHeader === NON_STATIC || + (showHeader !== undefined && typeof showHeader !== "boolean") || + headerTitle === NON_STATIC || + (headerTitle !== undefined && typeof headerTitle !== "string") + ) { + return dropNode(parent, index) + } + + const links: { label: string; url: string }[] = [] + for (const pageOrGroup of pages) { + const group = Array.isArray(pageOrGroup) ? pageOrGroup : [pageOrGroup] + if (group.length === 0) { + return dropNode(parent, index) + } + const groupPages: { name: string; url: string }[] = [] + for (const page of group) { + if ( + !page || + Array.isArray(page) || + typeof page !== "object" || + typeof page.name !== "string" || + typeof page.url !== "string" + ) { + return dropNode(parent, index) + } + groupPages.push({ name: page.name, url: page.url }) + } + links.push({ label: groupPages.map((page) => page.name).join(" / "), url: groupPages[0].url }) + } + + const replacement: Node[] = [] + if (showHeader !== false) + replacement.push(headingNode(2, typeof headerTitle === "string" ? headerTitle : "Guide Versions")) + if (links.length > 0) { + replacement.push({ + type: "list", + ordered: false, + children: links.map( + ({ label, url }) => + ({ + type: "listItem", + children: [paragraphNode([linkNode(label, url)])], + }) as Parent + ), + } as Parent) + } + parent.children.splice(index, 1, ...replacement) + return index + replacement.length +} + +type SlottedElement = { node: MdxJsxNode; parent: Parent; slot: string } + +function slottedElements(node: Parent): SlottedElement[] { + const elements: SlottedElement[] = [] + const collect = (parent: Parent) => { + for (const child of parent.children || []) { + if (child.type === "mdxJsxFlowElement" || child.type === "mdxJsxTextElement") { + const slot = staticAttribute(child as MdxJsxNode, "slot") + if (typeof slot === "string") elements.push({ node: child as MdxJsxNode, parent, slot }) + continue + } + if ((child as Parent).children) collect(child as Parent) + } + } + collect(node) + return elements +} + +export function handleTabs(node: MdxJsxNode, parent: Parent, index: number): number | void { + const tabs: { key: string; label: string }[] = [] + const panels = new Map() + + for (const { node: child, slot } of slottedElements(node as Parent)) { + if (slot.startsWith("tab.")) { + const label = staticChildrenText(child as Parent) + if (label !== NON_STATIC && label) tabs.push({ key: slot.slice(4), label }) + } else if (slot.startsWith("panel.")) { + panels.set(slot.slice(6), (child as Parent).children || []) + } + } + + const replacement: Node[] = [] + for (const tab of tabs) { + const panel = panels.get(tab.key) + if (!panel) continue + replacement.push(headingNode(3, tab.label), ...panel) + } + if (replacement.length === 0) { + return dropNode(parent, index) + } + parent.children.splice(index, 1, ...replacement) + return index +} + +export function handlePackageManagerTabs(node: MdxJsxNode, parent: Parent, index: number): number | void { + const slots = slottedElements(node as Parent) + const replacement: Node[] = [] + for (const manager of ["npm", "yarn"]) { + const content = slots.find(({ slot }) => slot === manager)?.node as Parent | undefined + if (content) replacement.push(headingNode(3, manager), ...(content.children || [])) + } + if (replacement.length === 0) { + return dropNode(parent, index) + } + parent.children.splice(index, 1, ...replacement) + return index +} + +export function handleFragment(node: MdxJsxNode, parent: Parent, index: number): number | void { + const children = (node as Parent).children || [] + parent.children.splice(index, 1, ...children) + return index +} + +export function handleAccordion(node: MdxJsxNode, parent: Parent, index: number): number | void { + const title = staticAttribute(node, "title") + const number = staticAttribute(node, "number") + const titleSlot = slottedElements(node as Parent).find(({ slot }) => slot === "title") + const slotTitle = titleSlot ? staticChildrenText(titleSlot.node as Parent) : undefined + if ( + title === NON_STATIC || + typeof title !== "string" || + number === NON_STATIC || + (number !== undefined && typeof number !== "number") || + slotTitle === NON_STATIC || + (titleSlot && !slotTitle) + ) { + return dropNode(parent, index) + } + if (titleSlot) { + titleSlot.parent.children.splice(titleSlot.parent.children.indexOf(titleSlot.node), 1) + } + const label = `${typeof number === "number" ? `${number}. ` : ""}${slotTitle || title}` + const replacement: Node[] = [headingNode(3, label), ...((node as Parent).children || [])] + parent.children.splice(index, 1, ...replacement) + return index +} + +export function handleAddress(node: MdxJsxNode, parent: Parent, index: number): number | void { + const contractUrl = staticAttribute(node, "contractUrl") + const address = staticAttribute(node, "address") + const endLength = staticAttribute(node, "endLength") + if ( + contractUrl === NON_STATIC || + typeof contractUrl !== "string" || + address === NON_STATIC || + (address !== undefined && typeof address !== "string") || + endLength === NON_STATIC || + (endLength !== undefined && (typeof endLength !== "number" || !Number.isInteger(endLength) || endLength < 0)) + ) { + return dropNode(parent, index) + } + + const value = typeof address === "string" && address ? address : contractUrl.split("/").pop() || contractUrl + const display = + typeof endLength === "number" && endLength > 0 + ? `${value.slice(0, endLength + 2)}...${value.slice(-endLength)}` + : value + parent.children[index] = linkNode(display, contractUrl) +} + +export function handleCallout( + node: MdxJsxNode, + parent: Parent, + index: number, + context: ComponentContext +): number | void { + return handleAside(node, parent, index, context) +} + +const SELECTOR_COMPONENTS = { + AnyApiCallout: { astro: "src/features/any-api/common/AnyApiCallout.astro", attribute: "callout" }, + FeedsCommonCallout: { astro: "src/features/feeds/callouts/FeedsCommonCallout.astro", attribute: "callout" }, + ResourcesCallout: { astro: "src/features/resources/callouts/ResourcesCallout.astro", attribute: "callout" }, + DataStreams: { astro: "src/features/data-streams/common/DataStreams.astro", attribute: "section" }, +} as const + +function handleAstroSelector( + node: MdxJsxNode, + parent: Parent, + index: number, + context: ComponentContext, + componentName: keyof typeof SELECTOR_COMPONENTS +): number | void { + const config = SELECTOR_COMPONENTS[componentName] + const selector = staticAttribute(node, config.attribute) + if (typeof selector !== "string") { + return dropNode(parent, index) + } + + try { + const astroPath = resolveExistingWithin(process.cwd(), config.astro) + if (!astroPath) { + return dropNode(parent, index) + } + const astroDirectory = path.dirname(astroPath) + const source = fs.readFileSync(astroPath, "utf-8") + const imports = readStaticDefaultImports(source) + const conditions = readStaticJsxSelectorConditions(source, config.attribute) + + const importPath = imports.get(conditions.get(selector) || "") + const markdownPath = importPath && resolveExistingWithin(astroDirectory, importPath) + if (!markdownPath || path.extname(markdownPath) !== ".mdx") { + return dropNode(parent, index) + } + const markdown = removeLeadingMdxFrontmatter(fs.readFileSync(markdownPath, "utf-8")) + const tree = context.processor.parse(markdown) as Parent + parent.children.splice(index, 1, ...(tree.children || [])) + return index + } catch (e) { + console.warn(`Failed to process ${componentName} in ${context.mdxAbsPath}:`, e) + return dropNode(parent, index) + } +} + +export function handleAnyApiCallout( + node: MdxJsxNode, + parent: Parent, + index: number, + context: ComponentContext +): number | void { + return handleAstroSelector(node, parent, index, context, "AnyApiCallout") +} + +export function handleFeedsCommonCallout( + node: MdxJsxNode, + parent: Parent, + index: number, + context: ComponentContext +): number | void { + return handleAstroSelector(node, parent, index, context, "FeedsCommonCallout") +} + +export function handleResourcesCallout( + node: MdxJsxNode, + parent: Parent, + index: number, + context: ComponentContext +): number | void { + return handleAstroSelector(node, parent, index, context, "ResourcesCallout") +} + +export function handleDataStreams( + node: MdxJsxNode, + parent: Parent, + index: number, + context: ComponentContext +): number | void { + return handleAstroSelector(node, parent, index, context, "DataStreams") +} + +function escapeTableCell(value: string): string { + return value.replace(/\\/g, "\\\\").replace(/\|/g, "\\|").replace(/\r?\n/g, " ") +} + +export function handleSchemaFieldsTable( + node: MdxJsxNode, + parent: Parent, + index: number, + context: ComponentContext +): number | void { + const schema = staticAttribute(node, "schema") + const definition = typeof schema === "string" ? REPORT_SCHEMA_DEFINITIONS[schema] : undefined + if (!definition) { + return dropNode(parent, index) + } + const rows = [ + "| Field | Type | Description |", + "| --- | --- | --- |", + ...definition.fields.map((field) => { + const description = `${field.description}${field.link ? ` — [${field.link.label}](${field.link.href})` : ""}` + return `| \`${escapeTableCell(field.field)}\` | \`${escapeTableCell(field.type)}\` | ${escapeTableCell(description)} |` + }), + ] + const tree = context.processor.parse(rows.join("\n")) as Parent + parent.children.splice(index, 1, ...(tree.children || [])) + return index + (tree.children?.length || 0) +} diff --git a/src/lib/markdown/index.ts b/src/lib/markdown/index.ts index f3342c8a4fa..a690d676a22 100644 --- a/src/lib/markdown/index.ts +++ b/src/lib/markdown/index.ts @@ -4,3 +4,5 @@ */ export * from "./formatters.js" +export * from "./buildMarkdownArtifact.js" +export type { MarkdownArtifact } from "./types.js" diff --git a/src/lib/markdown/sourceScanners.ts b/src/lib/markdown/sourceScanners.ts new file mode 100644 index 00000000000..71c8488f208 --- /dev/null +++ b/src/lib/markdown/sourceScanners.ts @@ -0,0 +1,190 @@ +const HIGHLIGHTER_MARKERS = ["highlight-line", "highlight-start", "highlight-end"] as const + +function isWhitespace(code: number): boolean { + return code === 9 || code === 10 || code === 11 || code === 12 || code === 13 || code === 32 +} + +function isIdentifierStart(code: number): boolean { + return code === 36 || code === 95 || (code >= 65 && code <= 90) || (code >= 97 && code <= 122) +} + +function isIdentifierPart(code: number): boolean { + return isIdentifierStart(code) || (code >= 48 && code <= 57) +} + +function skipWhitespace(source: string, cursor: number): number { + while (cursor < source.length && isWhitespace(source.charCodeAt(cursor))) cursor += 1 + return cursor +} + +function findIdentifierToken(source: string, token: string, cursor: number): number { + while (cursor < source.length) { + const start = source.indexOf(token, cursor) + if (start < 0) return -1 + const end = start + token.length + if ( + (start === 0 || !isIdentifierPart(source.charCodeAt(start - 1))) && + (end === source.length || !isIdentifierPart(source.charCodeAt(end))) + ) { + return start + } + cursor = end + } + return -1 +} + +function readIdentifier(source: string, cursor: number): { value: string; end: number } | undefined { + if (!isIdentifierStart(source.charCodeAt(cursor))) return + const start = cursor + cursor += 1 + while (cursor < source.length && isIdentifierPart(source.charCodeAt(cursor))) cursor += 1 + return { value: source.slice(start, cursor), end: cursor } +} + +function quotedEndOnLine(source: string, cursor: number, quote: number): number { + while (cursor < source.length && source.charCodeAt(cursor) !== 10) { + if (source.charCodeAt(cursor) === quote) return cursor + cursor += 1 + } + return -1 +} + +export function readStaticDefaultImports(source: string): Map { + const imports = new Map() + let cursor = 0 + + while (cursor < source.length) { + const start = findIdentifierToken(source, "import", cursor) + if (start < 0) break + cursor = start + "import".length + if (!isWhitespace(source.charCodeAt(cursor))) continue + + cursor = skipWhitespace(source, cursor) + const identifier = readIdentifier(source, cursor) + if (!identifier) continue + cursor = identifier.end + if (!isWhitespace(source.charCodeAt(cursor))) continue + + cursor = skipWhitespace(source, cursor) + if (!source.startsWith("from", cursor) || isIdentifierPart(source.charCodeAt(cursor + "from".length))) { + continue + } + cursor += "from".length + if (!isWhitespace(source.charCodeAt(cursor))) continue + + cursor = skipWhitespace(source, cursor) + const quote = source.charCodeAt(cursor) + if (quote !== 34 && quote !== 39) continue + cursor += 1 + const end = quotedEndOnLine(source, cursor, quote) + if (end < 0) { + const lineEnd = source.indexOf("\n", cursor) + cursor = lineEnd < 0 ? source.length : lineEnd + 1 + continue + } + if (end > cursor) imports.set(identifier.value, source.slice(cursor, end)) + cursor = end + 1 + } + + return imports +} + +export function readStaticJsxSelectorConditions(source: string, attribute: string): Map { + const conditions = new Map() + if (attribute.length === 0) return conditions + let cursor = 0 + + while (cursor < source.length) { + const start = findIdentifierToken(source, attribute, cursor) + if (start < 0) break + cursor = start + attribute.length + cursor = skipWhitespace(source, cursor) + if (!source.startsWith("===", cursor)) continue + + cursor += 3 + cursor = skipWhitespace(source, cursor) + const quote = source.charCodeAt(cursor) + if (quote !== 34 && quote !== 39) continue + cursor += 1 + const selectorEnd = quotedEndOnLine(source, cursor, quote) + if (selectorEnd < 0) { + const lineEnd = source.indexOf("\n", cursor) + cursor = lineEnd < 0 ? source.length : lineEnd + 1 + continue + } + const selector = source.slice(cursor, selectorEnd) + cursor = skipWhitespace(source, selectorEnd + 1) + if (!source.startsWith("&&", cursor)) continue + + cursor += 2 + cursor = skipWhitespace(source, cursor) + if (source.charCodeAt(cursor) !== 60) continue + cursor += 1 + const component = readIdentifier(source, cursor) + if (!component) continue + cursor = component.end + if (selector.length > 0) conditions.set(selector, component.value) + } + + return conditions +} + +function isFenceLine(source: string, start: number, end: number): boolean { + if (source.charCodeAt(start) !== 45 || source.charCodeAt(start + 1) !== 45 || source.charCodeAt(start + 2) !== 45) { + return false + } + for (let cursor = start + 3; cursor < end; cursor += 1) { + if (!isWhitespace(source.charCodeAt(cursor))) return false + } + return true +} + +export function removeLeadingMdxFrontmatter(source: string): string { + const firstLineEnd = source.indexOf("\n") + if (firstLineEnd < 0 || !isFenceLine(source, 0, firstLineEnd)) return source + + let cursor = firstLineEnd + 1 + while (cursor < source.length) { + const lineEnd = source.indexOf("\n", cursor) + if (lineEnd < 0) return source + if (isFenceLine(source, cursor, lineEnd)) return source.slice(lineEnd + 1) + cursor = lineEnd + 1 + } + return source +} + +function highlighterMarkerLength(line: string, cursor: number): number { + for (const marker of HIGHLIGHTER_MARKERS) { + if (line.startsWith(marker, cursor)) return marker.length + } + return 0 +} + +function stripHighlighterCommentLine(line: string): string { + let cursor = 0 + let whitespaceStart = 0 + + while (cursor < line.length) { + if (isWhitespace(line.charCodeAt(cursor))) { + cursor += 1 + continue + } + if (line.charCodeAt(cursor) === 47 && line.charCodeAt(cursor + 1) === 47) { + const markerStart = skipWhitespace(line, cursor + 2) + const markerLength = highlighterMarkerLength(line, markerStart) + if (markerLength > 0) { + return line.slice(0, whitespaceStart) + line.slice(markerStart + markerLength) + } + } + cursor += 1 + whitespaceStart = cursor + } + + return line +} + +export function stripHighlighterComments(code: string): string { + const chunks: string[] = [] + for (const line of code.split("\n")) chunks.push(stripHighlighterCommentLine(line)) + return chunks.join("\n") +} diff --git a/src/lib/markdown/transformMarkdown.ts b/src/lib/markdown/transformMarkdown.ts index 58c20f5da64..d9e51cd8df7 100644 --- a/src/lib/markdown/transformMarkdown.ts +++ b/src/lib/markdown/transformMarkdown.ts @@ -20,63 +20,27 @@ import { handleClickToZoom, handleCodeSample, handleBilling, + handlePageTabs, + handleTabs, + handlePackageManagerTabs, + handleFragment, + handleAccordion, + handleAddress, + handleCallout, + handleAnyApiCallout, + handleFeedsCommonCallout, + handleResourcesCallout, + handleDataStreams, + handleSchemaFieldsTable, loadCcipCommonMapping, } from "./componentHandlers.js" import fs from "fs" import path from "path" - -/** - * Convert Aside components to markdown blockquotes - * Handles multi-line Aside tags by converting them to blockquote format - * Preserves Asides with nested JSX components (they'll be handled by AST or remain as-is) - * @param content - Markdown content that may contain Aside components - * @returns Content with simple Aside tags converted to blockquotes - */ -function convertAsidesToBlockquotes(content: string): string { - // Match multi-line Aside components - const asideRegex = /([\s\S]*?)<\/Aside>/g - - return content.replace(asideRegex, (fullMatch, type, title, children) => { - // Check if the Aside contains other JSX components (like Tabs, CopyText, etc.) - const hasJSXComponents = /<[A-Z]\w+/.test(children) - - if (hasJSXComponents) { - // Keep as-is - these complex nested structures need manual handling - // or will be dropped by the AST handlers - return fullMatch - } - - // Create a blockquote directly in markdown format - // This avoids JSX parsing issues entirely - const cleanChildren = children.trim() - const asideType = type.toUpperCase() - const header = title ? `**${asideType}: ${title}**` : `**${asideType}**` - - // Return as markdown blockquote - return `\n\n> ${header}\n>\n> ${cleanChildren}\n\n` - }) -} - -/** - * Convert ClickToZoom components to markdown images - * Handles self-closing ClickToZoom tags by converting to standard markdown image syntax - * @param content - Markdown content that may contain ClickToZoom components - * @returns Content with ClickToZoom tags converted to markdown images - */ -function convertClickToZoomToImages(content: string): string { - // Match self-closing ClickToZoom tags with any attributes - // Captures src and alt, ignores other attributes like style - const clickToZoomRegex = /]*src="([^"]+)"[^>]*(?:alt="([^"]*)")?[^>]*\/>/g - - return content.replace(clickToZoomRegex, (_, src, alt) => { - const altText = alt || "Image" - return `![${altText}](${src})` - }) -} +import { removeLeadingMdxFrontmatter } from "./sourceScanners.js" /** * Preprocess CcipCommon components by inlining their content - * This is essential because remarkMdx doesn't always parse self-closing JSX tags properly + * Inlined MDX components continue through the normal remark AST visitor * @param markdown - Raw markdown content * @returns Markdown with CcipCommon components replaced by their content */ @@ -90,22 +54,24 @@ function preprocessCcipCommon(markdown: string): string { const fileName = calloutFileMap[calloutName] if (fileName) { - const calloutPath = path.resolve("src/features/ccip", fileName) - if (fs.existsSync(calloutPath)) { + let calloutPath: string | undefined + try { + const ccipRoot = fs.realpathSync(path.resolve("src/features/ccip")) + const candidate = fs.realpathSync(path.resolve(ccipRoot, fileName)) + if (candidate === ccipRoot || candidate.startsWith(ccipRoot + path.sep)) calloutPath = candidate + } catch { + // Missing or escaping selector targets remain unexpanded and are dropped by the AST visitor. + } + if (calloutPath) { let calloutContent = fs.readFileSync(calloutPath, "utf-8") // Strip frontmatter if present - if (calloutContent.trim().startsWith("---")) { - calloutContent = calloutContent.replace(/^---\s*\n[\s\S]*?\n---\s*\n/, "") - } + calloutContent = removeLeadingMdxFrontmatter(calloutContent) // Strip import statements calloutContent = calloutContent.replace(/^import\s+.+$/gm, "").trim() - // Convert Aside components to blockquotes - calloutContent = convertAsidesToBlockquotes(calloutContent) - - // Replace the CcipCommon tag with the processed content + // Replace the CcipCommon tag with the inlined content preprocessedMarkdown = preprocessedMarkdown.replace(fullMatch, "\n\n" + calloutContent + "\n\n") } } @@ -128,18 +94,8 @@ export async function transformMarkdown( ): Promise { const { targetLanguage } = config - // Preprocessing pipeline - apply transformations before AST parsing - // This handles components that remarkMdx struggles to parse (multi-line JSX) - - // Step 1: Preprocess CcipCommon components (inline callout content) - let preprocessedMarkdown = preprocessCcipCommon(markdown) - - // Step 2: Convert Aside components to markdown blockquotes - // Applies to both main content and inlined CcipCommon content - preprocessedMarkdown = convertAsidesToBlockquotes(preprocessedMarkdown) - - // Step 3: Convert ClickToZoom to markdown images - preprocessedMarkdown = convertClickToZoomToImages(preprocessedMarkdown) + // Inline CcipCommon content before AST parsing so embedded components reach the normal AST handlers. + const preprocessedMarkdown = preprocessCcipCommon(markdown) // Create unified processor with remark plugins const processor = unified() @@ -179,7 +135,10 @@ export async function transformMarkdown( } // Handle ClickToZoom - if (node.type === "mdxJsxFlowElement" && (node as MdxJsxNode).name === "ClickToZoom") { + if ( + (node.type === "mdxJsxFlowElement" || node.type === "mdxJsxTextElement") && + (node as MdxJsxNode).name === "ClickToZoom" + ) { return handleClickToZoom(node as MdxJsxNode, parent, index, context) } @@ -190,7 +149,64 @@ export async function transformMarkdown( // Handle Billing if (node.type === "mdxJsxFlowElement" && (node as MdxJsxNode).name === "Billing") { - return handleBilling(node as MdxJsxNode, parent, index, context) + return handleBilling(parent, index, context) + } + + if (node.type === "mdxJsxFlowElement" && (node as MdxJsxNode).name === "PageTabs") { + return handlePageTabs(node as MdxJsxNode, parent, index) + } + + if ( + node.type === "mdxJsxFlowElement" && + ((node as MdxJsxNode).name === "Tabs" || (node as MdxJsxNode).name === "TabsContent") + ) { + return handleTabs(node as MdxJsxNode, parent, index) + } + + if (node.type === "mdxJsxFlowElement" && (node as MdxJsxNode).name === "PackageManagerTabs") { + return handlePackageManagerTabs(node as MdxJsxNode, parent, index) + } + + if (node.type === "mdxJsxFlowElement" && (node as MdxJsxNode).name === "Accordion") { + return handleAccordion(node as MdxJsxNode, parent, index) + } + + if (node.type === "mdxJsxFlowElement" && (node as MdxJsxNode).name === "Callout") { + return handleCallout(node as MdxJsxNode, parent, index, context) + } + + if (node.type === "mdxJsxFlowElement" && (node as MdxJsxNode).name === "AnyApiCallout") { + return handleAnyApiCallout(node as MdxJsxNode, parent, index, context) + } + + if (node.type === "mdxJsxFlowElement" && (node as MdxJsxNode).name === "FeedsCommonCallout") { + return handleFeedsCommonCallout(node as MdxJsxNode, parent, index, context) + } + + if (node.type === "mdxJsxFlowElement" && (node as MdxJsxNode).name === "ResourcesCallout") { + return handleResourcesCallout(node as MdxJsxNode, parent, index, context) + } + + if (node.type === "mdxJsxFlowElement" && (node as MdxJsxNode).name === "DataStreams") { + return handleDataStreams(node as MdxJsxNode, parent, index, context) + } + + if (node.type === "mdxJsxFlowElement" && (node as MdxJsxNode).name === "SchemaFieldsTable") { + return handleSchemaFieldsTable(node as MdxJsxNode, parent, index, context) + } + + if ( + (node.type === "mdxJsxFlowElement" || node.type === "mdxJsxTextElement") && + (node as MdxJsxNode).name === "Address" + ) { + return handleAddress(node as MdxJsxNode, parent, index) + } + + if ( + (node.type === "mdxJsxFlowElement" || node.type === "mdxJsxTextElement") && + (node as MdxJsxNode).name === "Fragment" + ) { + return handleFragment(node as MdxJsxNode, parent, index) } // Handle MDX JSX text elements @@ -208,36 +224,36 @@ export async function transformMarkdown( } } - // Drop MDX/import/export nodes (except handled components) + // Drop MDX/import/export nodes except the explicitly projected component names above. if ( - (node.type === "mdxJsxFlowElement" && + ((node.type === "mdxJsxFlowElement" || node.type === "mdxJsxTextElement") && (node as MdxJsxNode).name !== "Aside" && (node as MdxJsxNode).name !== "CcipCommon" && (node as MdxJsxNode).name !== "ClickToZoom" && (node as MdxJsxNode).name !== "CodeSample" && - (node as MdxJsxNode).name !== "Billing") || + (node as MdxJsxNode).name !== "Billing" && + (node as MdxJsxNode).name !== "PageTabs" && + (node as MdxJsxNode).name !== "Tabs" && + (node as MdxJsxNode).name !== "TabsContent" && + (node as MdxJsxNode).name !== "PackageManagerTabs" && + (node as MdxJsxNode).name !== "Fragment" && + (node as MdxJsxNode).name !== "Accordion" && + (node as MdxJsxNode).name !== "Address" && + (node as MdxJsxNode).name !== "Callout" && + (node as MdxJsxNode).name !== "AnyApiCallout" && + (node as MdxJsxNode).name !== "FeedsCommonCallout" && + (node as MdxJsxNode).name !== "ResourcesCallout" && + (node as MdxJsxNode).name !== "DataStreams" && + (node as MdxJsxNode).name !== "SchemaFieldsTable") || node.type === "mdxjsEsm" || node.type === "import" || - node.type === "export" - ) { - parent.children.splice(index, 1) - return - } - - // Handle HTML nodes - drop them - if (node.type === "html") { - parent.children.splice(index, 1) - return - } - - // Handle JSX comments - drop them - if ( - (node.type === "mdxFlowExpression" || node.type === "mdxTextExpression") && - typeof (node as { value?: string }).value === "string" && - (node as { value?: string }).value?.trim().match(/^\/\*[\s\S]*?\*\/$/) + node.type === "export" || + node.type === "mdxFlowExpression" || + node.type === "mdxTextExpression" || + node.type === "html" ) { parent.children.splice(index, 1) - return + return index } // Replace images with their alt text diff --git a/src/lib/markdown/types.ts b/src/lib/markdown/types.ts index 5362f96a33f..9548b2d5f59 100644 --- a/src/lib/markdown/types.ts +++ b/src/lib/markdown/types.ts @@ -110,3 +110,11 @@ export interface CodeBlock { /** Optional title */ title?: string } + +export type MarkdownArtifact = { + requestPath: string + routeKind: "normal" | "special" | "selector" | "redirect" + transformMode: "normal" | "sanitized" | "fallback" | "replacement" + sourcePath?: string + markdown: string +} diff --git a/src/pages/[...path].md.ts b/src/pages/[...path].md.ts index 1b4d916d93d..3c879aa92fd 100644 --- a/src/pages/[...path].md.ts +++ b/src/pages/[...path].md.ts @@ -1,29 +1,6 @@ import type { APIRoute } from "astro" -import fs from "node:fs/promises" -import path from "node:path" import { textPlainHeaders } from "@lib/api/cacheHeaders.js" -import { transformPageToMarkdown } from "@lib/markdown/transformMarkdown.js" -import { extractFrontmatter, getIsoStringOrUndefined, toCanonicalUrl, toContentRelative } from "@lib/markdown/utils.js" - -const SITE_BASE = "https://docs.chain.link" -const CONTENT_ROOT = path.resolve("src/content") - -const LLMS_DIRECTIVE = "> For the complete documentation index, see [llms.txt](/llms.txt)." - -const MARKDOWN_REDIRECTS: Record = { - "ccip/tutorials/cross-chain-tokens": "ccip/tutorials/evm/cross-chain-tokens", - - // Data Streams - "data-streams/getting-started": "data-streams/tutorials/streams-trade/getting-started", - "data-streams/getting-started-hardhat": "data-streams/tutorials/streams-trade/getting-started-hardhat", - "data-streams/reference/streams-direct/streams-direct-onchain-verification": - "data-streams/reference/onchain-verification", - - // Newly surfaced redirects - "chainlink-functions/resources/concepts": "chainlink-functions/resources", - "cre/getting-started/conclusion": "cre/getting-started", - "data-streams/reference/streams-direct/streams-direct-interface-ws": "data-streams/reference/interface-ws", -} +import { buildMarkdownArtifact } from "@lib/markdown/buildMarkdownArtifact.js" const markdownHeaders = { ...textPlainHeaders, @@ -33,296 +10,19 @@ const markdownHeaders = { export const prerender = false export const GET: APIRoute = async ({ params, request }) => { - const cleanPath = normalizeMarkdownPath(params.path) - - if (!cleanPath) { + const requestPath = params.path + if (!requestPath) { return new Response("Page not found.", { status: 404 }) } - const specialResolution = await resolveSpecialCanonicalMarkdownPath(cleanPath) - if (specialResolution) { - return buildMarkdownResponseFromPath(specialResolution.resolvedPath, request, specialResolution.sourceCanonicalPath) - } - - const creResolution = await resolveCreCanonicalMarkdownPath(cleanPath) - - if (creResolution.kind === "selector") { - return new Response(buildCreSelectorMarkdown(cleanPath, creResolution), { - status: 200, - headers: markdownHeaders, - }) - } - - const resolvedPath = creResolution.kind === "resolved" ? creResolution.path : cleanPath - return buildMarkdownResponseFromPath(resolvedPath, request) -} - -type SpecialResolution = { - resolvedPath: string - sourceCanonicalPath: string -} - -async function resolveSpecialCanonicalMarkdownPath(cleanPath: string): Promise { - const specialPathMap: Record = { - "cre-templates": "cre/templates", - } - - const mappedPath = specialPathMap[cleanPath] - if (!mappedPath) return null - - const file = await findContentFile(mappedPath) - if (!file) return null - - return { - resolvedPath: mappedPath, - sourceCanonicalPath: cleanPath, - } -} - -type CreResolution = - { kind: "none" } | { kind: "resolved"; path: string } | { kind: "selector"; goPath: string; tsPath: string } - -async function resolveCreCanonicalMarkdownPath(cleanPath: string): Promise { - if (!cleanPath.startsWith("cre/")) { - return { kind: "none" } - } - - const direct = await findContentFile(cleanPath) - if (direct) { - return { kind: "resolved", path: cleanPath } - } - - const goPath = `${cleanPath}-go` - const tsPath = `${cleanPath}-ts` - - const [goFile, tsFile] = await Promise.all([findContentFile(goPath), findContentFile(tsPath)]) - - if (goFile && tsFile) { - return { kind: "selector", goPath, tsPath } - } - - if (goFile) { - return { kind: "resolved", path: goPath } - } - - if (tsFile) { - return { kind: "resolved", path: tsPath } - } - - return { kind: "none" } -} - -async function buildMarkdownResponseFromPath( - resolvedPath: string, - request: Request, - sourceCanonicalPathOverride?: string -): Promise { - const redirectTarget = MARKDOWN_REDIRECTS[resolvedPath] - - if (redirectTarget) { - return buildMarkdownMovedResponse(resolvedPath, redirectTarget) - } - - const mdxAbsPath = await findContentFile(resolvedPath) - - if (!mdxAbsPath) { + const lang = new URL(request.url).searchParams.get("lang") || undefined + const artifact = await buildMarkdownArtifact(requestPath, { lang }) + if (!artifact) { return new Response("Page not found.", { status: 404 }) } - const url = new URL(request.url) - const targetLanguage = url.searchParams.get("lang") || undefined - - const raw = await fs.readFile(mdxAbsPath, "utf-8") - const { body, fmTitle, fmLastModified } = extractFrontmatter(raw) - - const section = resolvedPath.split("/")[0] - - const transformed = await transformPageBodyToMarkdown(body, mdxAbsPath, { - siteBase: SITE_BASE, - targetLanguage, - }) - - const relFromContent = toContentRelative(mdxAbsPath) - const derivedSourceUrl = toCanonicalUrl(section, relFromContent, SITE_BASE) - const sourceUrl = sourceCanonicalPathOverride ? `${SITE_BASE}/${sourceCanonicalPathOverride}` : derivedSourceUrl - - const title = fmTitle || path.basename(mdxAbsPath, path.extname(mdxAbsPath)) - const lastModified = getIsoStringOrUndefined(fmLastModified) - - const headerLines = [ - `# ${title}`, - `Source: ${sourceUrl}`, - ...(lastModified ? [`Last Updated: ${lastModified}`] : []), - "", - LLMS_DIRECTIVE, - "", - ] - - return new Response([...headerLines, transformed.trim()].join("\n"), { + return new Response(artifact.markdown, { status: 200, headers: markdownHeaders, }) } - -async function transformPageBodyToMarkdown( - body: string, - mdxAbsPath: string, - options: { - siteBase: string - targetLanguage?: string - } -): Promise { - // Targeted fix for problematic page - if (mdxAbsPath.includes("data-feeds/deprecating-feeds")) { - return ` -## Deprecated Feeds - -This page contains dynamically generated or component-heavy content. - -For the full and most up-to-date information, see: -https://docs.chain.link/data-feeds/deprecating-feeds -`.trim() - } - - try { - return await transformPageToMarkdown(body, mdxAbsPath, options) - } catch { - const sanitizedBody = stripRuntimeMdxSyntax(body) - - try { - return await transformPageToMarkdown(sanitizedBody, mdxAbsPath, options) - } catch { - return buildFallbackMarkdownBody(sanitizedBody) - } - } -} - -function buildFallbackMarkdownBody(body: string): string { - return stripRuntimeMdxSyntax(body) - .replace(/<([A-Z][A-Za-z0-9]*)\b[^>]*\/>/g, "") - .replace(/<([A-Z][A-Za-z0-9]*)\b[^>]*>/g, "") - .replace(/<\/[A-Z][A-Za-z0-9]*>/g, "") - .trim() -} - -function stripRuntimeMdxSyntax(body: string): string { - const lines = body.split("\n") - const output: string[] = [] - - let skippingExportBlock = false - let skippingImportBlock = false - let braceDepth = 0 - - for (const line of lines) { - const trimmed = line.trim() - - if (skippingImportBlock) { - if (trimmed.includes(" from ") || trimmed.endsWith('"') || trimmed.endsWith("'")) { - skippingImportBlock = false - } - continue - } - - if (skippingExportBlock) { - braceDepth += countChar(line, "{") - braceDepth -= countChar(line, "}") - - if (braceDepth <= 0) { - skippingExportBlock = false - braceDepth = 0 - } - continue - } - - if (/^import\s+/.test(trimmed)) { - if (!trimmed.includes(" from ")) skippingImportBlock = true - continue - } - - if (/^export\s+(async\s+)?function\s+/.test(trimmed)) { - skippingExportBlock = true - braceDepth = countChar(line, "{") - countChar(line, "}") - continue - } - - if (/^export\s+(const|let|var)\s+/.test(trimmed)) { - continue - } - - output.push(line) - } - - return output.join("\n") -} - -function countChar(value: string, char: string): number { - return value.split(char).length - 1 -} - -function normalizeMarkdownPath(pathParam: string | undefined): string | null { - if (!pathParam) return null - - const cleanPath = pathParam.replace(/\.md$/i, "").replace(/^\/+/, "").replace(/\/+$/, "") - - if (!cleanPath) return null - - const segments = cleanPath.split("/") - if (segments.some((segment) => segment === ".." || segment === "." || segment === "")) { - return null - } - - return cleanPath -} - -async function findContentFile(cleanPath: string): Promise { - const possiblePaths = [ - path.resolve(CONTENT_ROOT, `${cleanPath}.mdx`), - path.resolve(CONTENT_ROOT, cleanPath, "index.mdx"), - path.resolve(CONTENT_ROOT, `${cleanPath}.md`), - path.resolve(CONTENT_ROOT, cleanPath, "index.md"), - ] - - for (const candidate of possiblePaths) { - if (!candidate.startsWith(`${CONTENT_ROOT}${path.sep}`)) continue - try { - await fs.access(candidate) - return candidate - } catch {} - } - - return null -} - -function buildMarkdownMovedResponse(sourcePath: string, targetPath: string): Response { - const sourceUrl = `${SITE_BASE}/${sourcePath}` - const targetUrl = `/${targetPath}.md` - - return new Response( - [ - `# Redirect`, - `Source: ${sourceUrl}`, - "", - LLMS_DIRECTIVE, - "", - "This page has moved.", - "", - `Use the current documentation: [${targetPath}](${targetUrl}).`, - "", - ].join("\n"), - { status: 200, headers: markdownHeaders } - ) -} - -function buildCreSelectorMarkdown(canonicalPath: string, resolution: any): string { - const canonicalUrl = `${SITE_BASE}/${canonicalPath}` - return [ - `# ${canonicalPath}`, - `Source: ${canonicalUrl}`, - "", - LLMS_DIRECTIVE, - "", - `- Go: /${resolution.goPath}.md`, - `- TypeScript: /${resolution.tsPath}.md`, - "", - ].join("\n") -} diff --git a/src/scripts/check-markdown-fidelity.test.ts b/src/scripts/check-markdown-fidelity.test.ts new file mode 100644 index 00000000000..7888a09887f --- /dev/null +++ b/src/scripts/check-markdown-fidelity.test.ts @@ -0,0 +1,687 @@ +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, expect, test } from "@jest/globals" +import { buildMarkdownArtifact } from "@lib/markdown/buildMarkdownArtifact.js" +import type { MarkdownArtifact } from "@lib/markdown/types.js" +import { + analyzeSourceMarkdown, + compareSourceToArtifact, + checkPath, + createReport, + determineExitCode, + findingIdentity, + parseCliArguments, + readStaticExpression, + inspectSyntheticArtifact, + runMarkdownFidelity, + serializeReport, + type FidelityException, + type FidelityFinding, +} from "./check-markdown-fidelity.js" + +function artifact(markdown: string): MarkdownArtifact { + return { + requestPath: "fixture", + routeKind: "normal", + transformMode: "normal", + sourcePath: path.resolve("src/content/fixture.mdx"), + markdown, + } +} + +function syntheticArtifact(routeKind: "redirect" | "selector", markdown: string): MarkdownArtifact { + return { + requestPath: "fixture", + routeKind, + transformMode: "normal", + markdown, + } +} + +function finding(status: FidelityFinding["status"], occurrence: string, sourceLine = 1): FidelityFinding { + return { path: "fixture", status, occurrence, sourceLine } +} + +describe("Markdown fidelity execution modes", () => { + test("full-corpus reports findings without blocking", () => { + expect(determineExitCode("full-corpus", [finding("missing", "lang=default;fact=1;text=known")])).toBe(0) + }) + + test("--path is repeatable and blocks on every non-exempt failure", () => { + expect(parseCliArguments(["--path", "cre/example", "--path", "ccip/example"])).toEqual({ + mode: "focused", + paths: ["ccip/example", "cre/example"], + }) + expect(determineExitCode("focused", [finding("degraded", "lang=default;transform=fallback")])).toBe(1) + expect(determineExitCode("focused", [finding("unsupported", "lang=default;diagnostic=1;Widget")])).toBe(1) + }) + + test.each([ + ["src/content/cre/getting-started/cli-installation/index.mdx", "cre/getting-started/cli-installation"], + [ + "src/content/cre/getting-started/cli-installation/macos-linux.mdx", + "cre/getting-started/cli-installation/macos-linux", + ], + ["src/content/cre/getting-started/cli-installation/windows.mdx", "cre/getting-started/cli-installation/windows"], + ])("maps source path %s to production request path", (sourcePath, requestPath) => { + expect(parseCliArguments(["--path", sourcePath])).toEqual({ mode: "focused", paths: [requestPath] }) + }) + + test.each([ + "/Users/example/src/content/cre/page.mdx", + "../src/content/cre/page.mdx", + "src/content/../secrets.mdx", + "src/other/page.mdx", + "other/page.mdx", + "src/content/cre/page.txt", + "src/content/cre/page", + "cre/example.md", + "cre/example.md.md", + "cre/example.mdx", + "src/content/cre/page.md.md", + "src/content/cre/page.mdx.md", + ])("rejects unsafe or unsupported source path %s", (sourcePath) => { + expect(() => parseCliArguments(["--path", sourcePath])).toThrow(`Invalid Markdown path: ${sourcePath}`) + }) + + test.each([ + ["src/content/cre/getting-started/cli-installation/index.mdx", "cre/getting-started/cli-installation"], + [ + "src/content/cre/getting-started/cli-installation/macos-linux.mdx", + "cre/getting-started/cli-installation/macos-linux", + ], + ["src/content/cre/getting-started/cli-installation/windows.mdx", "cre/getting-started/cli-installation/windows"], + ])("checks source path %s through its production artifact", async (sourcePath, requestPath) => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "markdown-fidelity-")) + const reportPath = path.join(directory, "report.json") + try { + const { report } = await runMarkdownFidelity(["--path", sourcePath], { reportPath }) + + expect(report.pathCount).toBe(1) + expect(report.findings.length).toBeGreaterThan(0) + expect(report.findings.every((candidate) => candidate.path === requestPath)).toBe(true) + expect(report.findings.some((candidate) => candidate.occurrence === "lang=default;artifact")).toBe(false) + expect(JSON.parse(await fs.readFile(reportPath, "utf8"))).toMatchObject({ pathCount: 1 }) + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + test("an exact exception passes and does not cover another occurrence", () => { + const source = "\n" + const initial = compareSourceToArtifact("fixture", "src/content/fixture.mdx", source, artifact("")) + const first = initial[0] + const exception: FidelityException = { + path: first.path, + occurrence: first.occurrence, + status: "unsupported", + reason: "Known projection gap", + owner: "docs-platform", + removalCondition: "Remove when UnknownWidget has a static projection", + } + const checked = compareSourceToArtifact("fixture", "src/content/fixture.mdx", source, artifact(""), "default", [ + exception, + ]) + + expect(checked[0].exception).toEqual({ + reason: exception.reason, + owner: exception.owner, + removalCondition: exception.removalCondition, + }) + expect(checked[1].exception).toBeUndefined() + expect(determineExitCode("focused", [checked[0]])).toBe(0) + expect(determineExitCode("focused", checked)).toBe(1) + }) +}) + +describe("raw source analysis", () => { + test("unsupported findings preserve component name, repository path, and original line", () => { + const source = [ + "---", + "title: Fixture", + "---", + "", + "Visible text", + "", + 'lost', + ].join("\n") + const [unsupported] = compareSourceToArtifact( + "fixture", + "src/content/fixture.mdx", + source, + artifact("Visible text") + ).filter((candidate) => candidate.status === "unsupported") + + expect(unsupported).toMatchObject({ + name: "UnknownWidget", + sourcePath: "src/content/fixture.mdx", + sourceLine: 7, + sourceText: 'lost', + }) + }) + + test("CRE_CLI_VERSION remains unverifiable", () => { + const source = 'export const CRE_CLI_VERSION = VERSIONS["cre-cli"].LATEST\n\nCurrent version: {CRE_CLI_VERSION}' + const diagnostics = analyzeSourceMarkdown(source).diagnostics + + expect(diagnostics).toEqual( + expect.arrayContaining([expect.objectContaining({ status: "unverifiable", name: "CRE_CLI_VERSION", line: 3 })]) + ) + }) + + test("allowlists literals, arrays, and objects without executing dynamic syntax", () => { + expect( + readStaticExpression({ + type: "ArrayExpression", + elements: [ + { type: "Literal", value: "go" }, + { + type: "ObjectExpression", + properties: [ + { + type: "Property", + computed: false, + kind: "init", + key: { type: "Identifier", name: "name" }, + value: { type: "Literal", value: "TypeScript" }, + }, + ], + }, + ], + }) + ).toEqual({ ok: true, value: ["go", { name: "TypeScript" }] }) + expect( + readStaticExpression({ type: "CallExpression", callee: { type: "Identifier", name: "sideEffect" } }) + ).toEqual({ + ok: false, + syntax: "CallExpression", + }) + expect(readStaticExpression({ type: "MemberExpression" })).toEqual({ ok: false, syntax: "MemberExpression" }) + expect(readStaticExpression({ type: "NewExpression" })).toEqual({ ok: false, syntax: "NewExpression" }) + }) + + test("enumerates every static language key without evaluating imported code identifiers", () => { + const source = [ + "", + "", + ].join("\n") + const analysis = analyzeSourceMarkdown(source) + + expect(analysis.languages).toEqual(["go", "ts"]) + expect(analysis.diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: "CodeHighlightBlockMulti.languages", status: "unverifiable" }), + expect.objectContaining({ name: "CodeHighlightBlockMulti.languages.go", status: "unverifiable" }), + expect.objectContaining({ name: "CodeHighlightBlockMulti.languages.ts", status: "unverifiable" }), + ]) + ) + }) + + test("code fences containing JSX-like text are ordinary code facts", () => { + const analysis = analyzeSourceMarkdown("```tsx\n{dangerous()}\n```") + + expect(analysis.diagnostics).toEqual([]) + expect(analysis.facts).toEqual([ + expect.objectContaining({ kind: "code", value: "{dangerous()}" }), + ]) + }) +}) + +describe("PageTabs occurrence grouping", () => { + test.each(["index.mdx", "macos-linux.mdx", "windows.mdx"])( + "three CLI pages PageTabs are present with grouped macOS / Linux then Windows: %s", + async (fileName) => { + const sourcePath = path.join("src/content/cre/getting-started/cli-installation", fileName) + const source = await fs.readFile(sourcePath, "utf8") + const served = [ + "## Select your operating system", + "", + "- [macOS / Linux](/cre/getting-started/cli-installation/macos-linux)", + "- [Windows](/cre/getting-started/cli-installation/windows)", + ].join("\n") + const findings = compareSourceToArtifact(sourcePath, sourcePath, source, artifact(served)) + const pageTabs = findings.filter( + (candidate) => + candidate.expected === "Select your operating system" || + candidate.expected === "macOS / Linux -> /cre/getting-started/cli-installation/macos-linux" || + candidate.expected === "Windows -> /cre/getting-started/cli-installation/windows" + ) + + expect(pageTabs.map((candidate) => [candidate.status, candidate.expected])).toEqual([ + ["present", "Select your operating system"], + ["present", "macOS / Linux -> /cre/getting-started/cli-installation/macos-linux"], + ["present", "Windows -> /cre/getting-started/cli-installation/windows"], + ]) + } + ) + + test("ordered matching rejects a removed duplicate and swapped links", () => { + const duplicateSource = "same\n\nsame" + const duplicateFindings = compareSourceToArtifact( + "fixture", + "src/content/fixture.mdx", + duplicateSource, + artifact("same") + ) + expect(duplicateFindings.map((candidate) => candidate.status)).toEqual(["present", "missing"]) + + const tabs = `` + const swapped = "[Second](/second)\n\n[First](/first)" + const swappedFindings = compareSourceToArtifact("fixture", "src/content/fixture.mdx", tabs, artifact(swapped)) + expect(swappedFindings.map((candidate) => candidate.status)).toEqual(["present", "missing"]) + }) +}) + +describe("source-less artifact fidelity", () => { + test("redirects require the exact current link and inspect their target page", async () => { + const requestPath = "data-streams/reference/streams-direct/streams-direct-interface-ws" + const targetPath = "data-streams/reference/interface-ws" + const correct = inspectSyntheticArtifact( + requestPath, + syntheticArtifact("redirect", `[${targetPath}](/${targetPath}.md)`) + ) + const wrong = inspectSyntheticArtifact(requestPath, syntheticArtifact("redirect", `[${targetPath}](/wrong.md)`)) + const empty = inspectSyntheticArtifact(requestPath, syntheticArtifact("redirect", "")) + + expect(correct.targetPaths).toEqual([targetPath]) + expect(correct.findings).toEqual([ + expect.objectContaining({ + status: "present", + occurrence: `lang=default;synthetic=redirect;${targetPath} -> /${targetPath}.md`, + sourceLine: null, + }), + ]) + expect(wrong.findings[0]).toMatchObject({ status: "missing", sourceLine: null }) + expect(empty.findings[0]).toMatchObject({ status: "missing", sourceLine: null }) + + const evaluated = await checkPath(requestPath) + expect(evaluated.some((candidate) => candidate.path === targetPath)).toBe(true) + }) + + test("CRE selectors require both exact entries and inspect both target pages", async () => { + const requestPath = "cre/reference/sdk/evm-client" + const goPath = `${requestPath}-go` + const tsPath = `${requestPath}-ts` + const correct = inspectSyntheticArtifact( + requestPath, + syntheticArtifact("selector", `- Go: /${goPath}.md\n- TypeScript: /${tsPath}.md`) + ) + const wrong = inspectSyntheticArtifact( + requestPath, + syntheticArtifact("selector", `- Go: /wrong.md\n- TypeScript: /${tsPath}.md`) + ) + const empty = inspectSyntheticArtifact(requestPath, syntheticArtifact("selector", "")) + + expect(correct.targetPaths).toEqual([goPath, tsPath]) + expect(correct.findings.map((candidate) => candidate.status)).toEqual(["present", "present"]) + expect(wrong.findings.map((candidate) => candidate.status)).toEqual(["missing", "present"]) + expect(empty.findings.map((candidate) => candidate.status)).toEqual(["missing", "missing"]) + + const evaluated = await checkPath(requestPath) + const evaluatedPaths = new Set(evaluated.map((candidate) => candidate.path)) + expect(evaluatedPaths.has(goPath)).toBe(true) + expect(evaluatedPaths.has(tsPath)).toBe(true) + }) +}) + +describe("content-bearing component fidelity", () => { + test("static CodeHighlightBlock and CodeSample content cannot disappear", () => { + const codeHighlight = '' + const codeHighlightCorrect = compareSourceToArtifact( + "fixture", + "src/content/fixture.mdx", + codeHighlight, + artifact("Code snippet for fixture.ts:\n\n```ts\nconst answer = 42\n```") + ) + const codeHighlightDropped = compareSourceToArtifact( + "fixture", + "src/content/fixture.mdx", + codeHighlight, + artifact("") + ) + const codeSample = '' + const codeSampleLink = + "[Open APIConsumer.sol in Remix](https://remix.ethereum.org/#url=https://docs.chain.link/samples/APIRequests/APIConsumer.sol)" + + expect(codeHighlightCorrect.map((candidate) => candidate.status)).toEqual(["present", "present"]) + expect(codeHighlightDropped.map((candidate) => candidate.status)).toEqual(["missing", "missing"]) + expect( + compareSourceToArtifact("fixture", "src/content/fixture.mdx", codeSample, artifact(codeSampleLink)).map( + (candidate) => candidate.status + ) + ).toEqual(["present"]) + expect(compareSourceToArtifact("fixture", "src/content/fixture.mdx", codeSample, artifact(""))[0]).toMatchObject({ + status: "missing", + sourcePath: "src/content/fixture.mdx", + sourceLine: 1, + }) + }) + test("static SchemaFieldsTable facts are checked and the current projection passes", async () => { + const source = '' + const analysis = analyzeSourceMarkdown(source, "src/content/fixture.mdx") + + expect(analysis.diagnostics).toEqual([]) + expect(analysis.facts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: "text", value: "Field" }), + expect.objectContaining({ kind: "text", value: "feedId" }), + expect.objectContaining({ kind: "text", value: "price" }), + ]) + ) + expect( + compareSourceToArtifact("fixture", "src/content/fixture.mdx", source, artifact("")).some( + (candidate) => candidate.status === "missing" + ) + ).toBe(true) + + const current = (await checkPath("data-streams/reference/report-schema-v2")).filter( + (candidate) => + candidate.sourcePath === "src/content/data-streams/reference/report-schema-v2.mdx" && + candidate.sourceLine === 27 + ) + expect(current.length).toBeGreaterThan(3) + expect(current.every((candidate) => candidate.status === "present")).toBe(true) + }) + + test.each([ + ["CodeHighlightBlock", ""], + ["CodeSample", ""], + ["AnyApiCallout", ''], + ["CcipCommon", ''], + ["SchemaFieldsTable", ''], + ["Billing", ""], + ])("%s unresolved content is blocking with component, path, and line", (name, component) => { + const source = `Visible\n\n${component}` + const [diagnostic] = compareSourceToArtifact( + "fixture", + "src/content/fixture.mdx", + source, + artifact("Visible") + ).filter((candidate) => candidate.status === "unverifiable") + + expect(diagnostic).toMatchObject({ + name, + sourcePath: "src/content/fixture.mdx", + sourceLine: 3, + sourceText: component, + }) + expect(JSON.parse(findingIdentity(diagnostic))).toMatchObject({ + path: "fixture", + status: "unverifiable", + language: "default", + component: name, + reason: diagnostic.reason, + }) + }) + + test.each([ + ['', "Use Chainlink Functions"], + ['', "Best Practices"], + ])("static selector content is independently inventoried: %s", (component, expectedFragment) => { + const analysis = analyzeSourceMarkdown(component, "src/content/fixture.mdx") + + expect(analysis.diagnostics).toEqual([]) + expect(analysis.facts.some((fact) => fact.value.includes(expectedFragment))).toBe(true) + expect( + compareSourceToArtifact("fixture", "src/content/fixture.mdx", component, artifact("")).some( + (candidate) => candidate.status === "missing" + ) + ).toBe(true) + }) +}) + +describe("linked heading fidelity", () => { + test("compares the nested destination without duplicating heading text", () => { + const source = "# [Current guide](/current)" + const correct = compareSourceToArtifact("fixture", "src/content/fixture.mdx", source, artifact(source)) + const changed = compareSourceToArtifact( + "fixture", + "src/content/fixture.mdx", + source, + artifact("# [Current guide](/changed)") + ) + const analysis = analyzeSourceMarkdown(source) + + expect(analysis.facts.map((fact) => [fact.kind, fact.value])).toEqual([ + ["heading", "Current guide"], + ["link", "Current guide"], + ]) + expect(correct.map((candidate) => candidate.status)).toEqual(["present", "present"]) + expect(changed.map((candidate) => candidate.status)).toEqual(["present", "missing"]) + expect(changed[1]).toMatchObject({ + expected: "Current guide -> /current", + sourcePath: "src/content/fixture.mdx", + sourceLine: 1, + }) + }) +}) + +describe("semantic fact boundaries and identities", () => { + test("coalesces adjacent inline text on both sides of comparison", () => { + const source = "Install the **CRE CLI** now." + const analysis = analyzeSourceMarkdown(source) + const findings = compareSourceToArtifact( + "fixture", + "src/content/fixture.mdx", + source, + artifact("Install the CRE CLI now.") + ) + + expect(analysis.facts.map((fact) => [fact.kind, fact.value])).toEqual([["text", "Install the CRE CLI now."]]) + expect(findings.map((candidate) => candidate.status)).toEqual(["present"]) + }) + + test("keeps the real CLI dynamic expression as a text boundary", async () => { + const source = await fs.readFile("src/content/cre/reference/cli/index.mdx", "utf8") + const analysis = analyzeSourceMarkdown(source, "src/content/cre/reference/cli/index.mdx") + const line18 = analysis.facts.filter((fact) => fact.line === 18 && fact.kind === "text").map((fact) => fact.value) + const line19 = analysis.facts.filter((fact) => fact.line === 19 && fact.kind === "text").map((fact) => fact.value) + + expect(line18).toContain( + "To ensure compatibility with the guides and examples in this documentation, please use version" + ) + expect(line19).toContain( + "of the CRE CLI. You can check your installed version by running cre version. Refer to the" + ) + expect([...line18, ...line19].some((value) => value.includes("version of the CRE CLI"))).toBe(false) + }) + + test("blank lines and inserted distinct present facts do not change a loss identity", () => { + const original = compareSourceToArtifact( + "fixture", + "src/content/fixture.mdx", + "Visible\n\nLost", + artifact("Visible") + ).find((candidate) => candidate.status === "missing") + const blankLineInserted = compareSourceToArtifact( + "fixture", + "src/content/fixture.mdx", + "Visible\n\n\nLost", + artifact("Visible") + ).find((candidate) => candidate.status === "missing") + const presentFactInserted = compareSourceToArtifact( + "fixture", + "src/content/fixture.mdx", + "Inserted\n\nLost", + artifact("Inserted") + ).find((candidate) => candidate.status === "missing") + const withoutInsertedFact = compareSourceToArtifact( + "fixture", + "src/content/fixture.mdx", + "Lost", + artifact("") + ).find((candidate) => candidate.status === "missing") + + if (!original || !blankLineInserted || !presentFactInserted || !withoutInsertedFact) { + throw new Error("Expected missing findings") + } + expect(findingIdentity(original)).toBe(findingIdentity(blankLineInserted)) + expect(findingIdentity(presentFactInserted)).toBe(findingIdentity(withoutInsertedFact)) + }) + + test("long losses with a common display prefix retain distinct full identities", () => { + const prefix = "same-prefix-".repeat(10) + const findings = compareSourceToArtifact( + "fixture", + "src/content/fixture.mdx", + `${prefix}alpha\n\n${prefix}beta`, + artifact("") + ) + const identities = findings.map(findingIdentity) + + expect(findings.map((candidate) => candidate.display)).toEqual([ + expect.stringMatching(/\.\.\.$/), + expect.stringMatching(/\.\.\.$/), + ]) + expect(new Set(identities).size).toBe(2) + expect(identities[0]).toContain(`${prefix}alpha`) + expect(identities[1]).toContain(`${prefix}beta`) + }) + + test("residual findings retain exact served syntax and served line", () => { + const served = "Visible\n\n" + const [residual] = compareSourceToArtifact( + "fixture", + "src/content/fixture.mdx", + "Visible", + artifact(served) + ).filter((candidate) => candidate.occurrence.includes(";residual=")) + const identity = JSON.parse(findingIdentity(residual)) + + expect(residual).toMatchObject({ + status: "unverifiable", + name: "Widget", + reason: "Served Markdown contains residual runtime syntax", + servedLine: 3, + servedText: "", + }) + expect(residual.occurrence).toContain('"servedText":""') + expect(identity.servedText).toBe("") + }) + + test("globally scheduled or visited synthetic targets are not emitted recursively", async () => { + const requestPath = "cre/reference/sdk/evm-client" + const goPath = `${requestPath}-go` + const tsPath = `${requestPath}-ts` + const scheduled = await checkPath(requestPath, { + globallyScheduledPaths: new Set([requestPath, goPath, tsPath]), + }) + const visited = await checkPath(requestPath, { + globallyScheduledPaths: new Set([requestPath]), + globallyVisitedPaths: new Set([goPath, tsPath]), + }) + + expect(new Set(scheduled.map((candidate) => candidate.path))).toEqual(new Set([requestPath])) + expect(new Set(visited.map((candidate) => candidate.path))).toEqual(new Set([requestPath])) + expect(new Set(scheduled.map(findingIdentity)).size).toBe(scheduled.length) + expect(new Set(visited.map(findingIdentity)).size).toBe(visited.length) + }) +}) + +describe("final projection and envelope regressions", () => { + test("checks imported CodeHighlightBlockMulti branches on a current production page", async () => { + const findings = (await checkPath("cre")).filter( + (candidate) => + candidate.sourcePath === "src/content/cre/index.mdx" && + candidate.sourceLine === 70 && + candidate.occurrence.includes('"kind":"code"') + ) + + expect(findings.length).toBe(4) + expect(findings.every((candidate) => candidate.status === "present")).toBe(true) + expect(new Set(findings.map((candidate) => candidate.lang ?? "default"))).toEqual(new Set(["default", "go", "ts"])) + }) + + test("reports DownloadButton as an unsupported visible component", () => { + const source = 'Visible\n\nDownload the toolkit' + const [unsupported] = compareSourceToArtifact( + "fixture", + "src/content/fixture.mdx", + source, + artifact("Visible") + ).filter((candidate) => candidate.status === "unsupported") + + expect(unsupported).toMatchObject({ + name: "DownloadButton", + sourcePath: "src/content/fixture.mdx", + sourceLine: 3, + sourceText: 'Download the toolkit', + }) + }) + + test("requires the exact production title, Source URL, and llms.txt directive", async () => { + const requestPath = "cre/getting-started/cli-installation" + const sourcePath = "src/content/cre/getting-started/cli-installation/index.mdx" + const source = await fs.readFile(sourcePath, "utf8") + const production = await buildMarkdownArtifact(requestPath) + expect(production).not.toBeNull() + if (!production) throw new Error(`Expected production Markdown artifact for ${requestPath}`) + + const envelope = compareSourceToArtifact(requestPath, sourcePath, source, production).filter((candidate) => + candidate.name?.startsWith("Envelope.") + ) + expect(envelope.map((candidate) => [candidate.name, candidate.status])).toEqual([ + ["Envelope.title", "present"], + ["Envelope.source", "present"], + ["Envelope.directive", "present"], + ]) + + const mutations = [ + ["Envelope.title", production.markdown.replace(/^# .+$/m, "# Changed title")], + ["Envelope.source", production.markdown.replace(/^Source: .+$/m, "Source: https://example.test/changed")], + [ + "Envelope.directive", + production.markdown.replace( + "> For the complete documentation index, see [llms.txt](/llms.txt).", + "> Documentation index removed." + ), + ], + ] as const + for (const [name, markdown] of mutations) { + const [finding] = compareSourceToArtifact(requestPath, sourcePath, source, { ...production, markdown }).filter( + (candidate) => candidate.name === name + ) + expect(finding).toMatchObject({ status: "missing", name }) + } + }) + + test("numbers unchanged missing duplicates independently from identical present copies", () => { + const original = compareSourceToArtifact( + "fixture", + "src/content/fixture.mdx", + "same\n\nsame", + artifact("same") + ).find((candidate) => candidate.status === "missing") + const withPresentCopy = compareSourceToArtifact( + "fixture", + "src/content/fixture.mdx", + "same\n\nsame\n\nsame", + artifact("same\n\nsame") + ).find((candidate) => candidate.status === "missing") + + expect(original).toBeDefined() + expect(withPresentCopy).toBeDefined() + if (!original || !withPresentCopy) throw new Error("Expected one unchanged missing duplicate in both comparisons") + expect(original.occurrence).toContain(";duplicate=1") + expect(withPresentCopy.occurrence).toContain(";duplicate=1") + expect(findingIdentity(withPresentCopy)).toBe(findingIdentity(original)) + }) +}) + +describe("stable report JSON", () => { + test("is sorted, timestamp-free, and contains no score", () => { + const second = finding("unsupported", "lang=default;diagnostic=2;Second", 20) + const first = finding("missing", "lang=default;fact=1;text=First", 10) + const left = serializeReport(createReport(2, [second, first])) + const right = serializeReport(createReport(2, [first, second])) + + expect(left).toBe(right) + expect(left).not.toContain("score") + expect(left).not.toContain("timestamp") + expect(JSON.parse(left)).toMatchObject({ + pathCount: 2, + counts: { present: 0, missing: 1, unsupported: 1, unverifiable: 0, degraded: 0 }, + }) + }) +}) diff --git a/src/scripts/check-markdown-fidelity.ts b/src/scripts/check-markdown-fidelity.ts new file mode 100644 index 00000000000..3d4635ca0fe --- /dev/null +++ b/src/scripts/check-markdown-fidelity.ts @@ -0,0 +1,1885 @@ +import fs from "node:fs/promises" +import fsSync from "node:fs" +import path from "node:path" +import { fileURLToPath } from "node:url" +import { unified } from "unified" +import remarkGfm from "remark-gfm" +import remarkMdx from "remark-mdx" +import remarkParse from "remark-parse" +import { SKIP, visit } from "unist-util-visit" +import type { Node, Parent } from "unist" +import { buildMarkdownArtifact, normalizeMarkdownPath } from "@lib/markdown/buildMarkdownArtifact.js" +import { + readStaticDefaultImports, + readStaticJsxSelectorConditions, + stripHighlighterComments, +} from "@lib/markdown/sourceScanners.js" +import type { MarkdownArtifact } from "@lib/markdown/types.js" +import { markdownFidelityExceptions } from "./markdown-fidelity-exceptions.js" + +const CONTENT_ROOT = path.resolve("src/content") +const DEFAULT_REPORT_PATH = "reports/markdown-fidelity-report.json" +const SITE_BASE = "https://docs.chain.link" +const LLMS_DIRECTIVE = "> For the complete documentation index, see [llms.txt](/llms.txt)." + +const MARKDOWN_REDIRECT_TARGETS = { + "ccip/tutorials/cross-chain-tokens": "ccip/tutorials/evm/cross-chain-tokens", + "chainlink-functions/resources/concepts": "chainlink-functions/resources", + "cre/getting-started/conclusion": "cre/getting-started", + "data-streams/getting-started": "data-streams/tutorials/streams-trade/getting-started", + "data-streams/getting-started-hardhat": "data-streams/tutorials/streams-trade/getting-started-hardhat", + "data-streams/reference/streams-direct/streams-direct-interface-ws": "data-streams/reference/interface-ws", + "data-streams/reference/streams-direct/streams-direct-onchain-verification": + "data-streams/reference/onchain-verification", +} as const + +const MARKDOWN_REDIRECT_PATHS = Object.keys(MARKDOWN_REDIRECT_TARGETS) + +export type FidelityStatus = "present" | "missing" | "unsupported" | "unverifiable" | "degraded" +export type RunMode = "focused" | "full-corpus" + +export interface FidelityException { + path: string + occurrence: string + status: Exclude + reason: string + owner: string + removalCondition: string +} + +export interface FidelityFinding { + path: string + status: FidelityStatus + occurrence: string + sourcePath?: string + sourceLine: number | null + sourceText?: string + lang?: string + name?: string + expected?: string + reason?: string + exception?: Pick + servedLine?: number + servedText?: string + display?: string +} + +export interface FidelityReport { + pathCount: number + counts: Record + findings: FidelityFinding[] +} + +export interface SourceFact { + ordinal: number + kind: "text" | "heading" | "link" | "code" + value: string + url?: string + depth?: number + variant?: string + line: number + sourceText: string + sourcePath?: string +} + +export interface SourceDiagnostic { + status: "unsupported" | "unverifiable" + ordinal: number + name: string + line: number + sourceText: string + reason: string + sourcePath?: string +} + +export interface SourceAnalysis { + facts: SourceFact[] + diagnostics: SourceDiagnostic[] + languages: string[] +} + +interface ObservedFact { + kind: SourceFact["kind"] + value: string + url?: string + depth?: number +} + +interface ObservedAnalysis { + facts: ObservedFact[] + residuals: Array<{ name: string; line: number; text: string; reason: string }> +} + +type AstRecord = Record + +const processor = unified().use(remarkParse).use(remarkMdx).use(remarkGfm) +const containerElements: Record = { + div: true, + Fragment: true, +} + +const selectorComponents = { + AnyApiCallout: { astro: "src/features/any-api/common/AnyApiCallout.astro", attribute: "callout" }, + FeedsCommonCallout: { astro: "src/features/feeds/callouts/FeedsCommonCallout.astro", attribute: "callout" }, + ResourcesCallout: { astro: "src/features/resources/callouts/ResourcesCallout.astro", attribute: "callout" }, + DataStreams: { astro: "src/features/data-streams/common/DataStreams.astro", attribute: "section" }, + CcipCommon: { astro: "src/features/ccip/CcipCommon.astro", attribute: "callout" }, +} as const + +function normalizeText(value: string): string { + return value.replace(/\s+/g, " ").trim() +} + +type GroupedFact = T & { group?: string; rawValue?: string } + +function coalesceTextFacts(facts: GroupedFact[]): T[] { + const coalesced: GroupedFact[] = [] + for (const fact of facts) { + const previous = coalesced[coalesced.length - 1] + if (fact.kind === "text" && fact.group && previous?.kind === "text" && previous.group === fact.group) { + previous.rawValue = `${previous.rawValue ?? previous.value}${fact.rawValue ?? fact.value}` + previous.value = normalizeText(previous.rawValue) + continue + } + coalesced.push({ ...fact }) + } + return coalesced.map((fact) => { + const result = { ...fact } + delete result.group + delete result.rawValue + return result as T + }) +} + +function lineText(lines: string[], line: number): string { + return lines[line - 1] ?? "" +} + +function maskFrontmatter(source: string): string { + if (!source.startsWith("---\n") && !source.startsWith("---\r\n")) return source + const lines = source.split(/(?<=\n)/) + for (let index = 1; index < lines.length; index += 1) { + if (/^---\s*(?:\r?\n)?$/.test(lines[index])) { + return lines.map((line, lineIndex) => (lineIndex <= index ? line.replace(/[^\r\n]/g, " ") : line)).join("") + } + } + return source +} + +function nodeLine(node: Node): number { + return node.position?.start.line ?? 1 +} + +function childrenOf(node: Node): Node[] { + return Array.isArray((node as Parent).children) ? (node as Parent).children : [] +} + +function nodeVisibleText(node: Node): string { + if (node.type === "text" || node.type === "inlineCode") { + return String((node as Node & { value?: unknown }).value ?? "") + } + return childrenOf(node).map(nodeVisibleText).join("") +} + +function expressionFrom(value: unknown): AstRecord | null { + if (!value || typeof value !== "object") return null + const data = (value as AstRecord).data + const estree = data && typeof data === "object" ? (data as AstRecord).estree : undefined + const body = estree && typeof estree === "object" ? (estree as AstRecord).body : undefined + const statement = Array.isArray(body) ? body[0] : undefined + const expression = statement && typeof statement === "object" ? (statement as AstRecord).expression : undefined + return expression && typeof expression === "object" ? (expression as AstRecord) : null +} +function staticImports(tree: Node): Map { + const imports = new Map() + for (const node of childrenOf(tree)) { + if (node.type !== "mdxjsEsm") continue + const data = (node as Node & { data?: AstRecord }).data + const estree = data?.estree as AstRecord | undefined + const body = estree?.body + if (!Array.isArray(body)) continue + for (const statementValue of body) { + const statement = statementValue as AstRecord + const source = statement.source as AstRecord | undefined + if ( + statement.type !== "ImportDeclaration" || + typeof source?.value !== "string" || + !Array.isArray(statement.specifiers) + ) { + continue + } + for (const specifierValue of statement.specifiers) { + const specifier = specifierValue as AstRecord + const local = specifier.local as AstRecord | undefined + if ( + (specifier.type === "ImportDefaultSpecifier" || specifier.type === "ImportSpecifier") && + typeof local?.name === "string" + ) { + imports.set(local.name, source.value) + } + } + } + } + return imports +} + +function resolveProjectFile(candidate: string): { absolute: string; relative: string } | null { + try { + const root = fsSync.realpathSync(process.cwd()) + const absolute = fsSync.realpathSync(path.resolve(candidate)) + if (absolute !== root && !absolute.startsWith(`${root}${path.sep}`)) return null + if (!fsSync.statSync(absolute).isFile()) return null + return { absolute, relative: path.relative(process.cwd(), absolute).split(path.sep).join("/") } + } catch { + return null + } +} + +function sourceLocation(sourcePath: string | undefined): { absolute: string; relative: string } | null { + if (!sourcePath) return null + return resolveProjectFile(path.isAbsolute(sourcePath) ? sourcePath : path.resolve(sourcePath)) +} + +function expressionAttribute(node: Node, name: string): AstRecord | null { + const attributes = (node as Node & { attributes?: AstRecord[] }).attributes ?? [] + return expressionFrom(attributes.find((candidate) => candidate.name === name)?.value) +} + +function resolveSelectorTarget( + component: keyof typeof selectorComponents, + selector: string +): { target?: { absolute: string; relative: string }; reason?: string } { + const config = selectorComponents[component] + const astro = resolveProjectFile(config.astro) + if (!astro) return { reason: `Selector definition ${config.astro} is missing or escapes the project` } + const source = fsSync.readFileSync(astro.absolute, "utf8") + const imports = readStaticDefaultImports(source) + const componentBySelector = readStaticJsxSelectorConditions(source, config.attribute) + const importedPath = imports.get(componentBySelector.get(selector) ?? "") + if (!importedPath?.endsWith(".mdx")) + return { reason: `${component} selector "${selector}" has no static MDX target in ${astro.relative}` } + const target = resolveProjectFile(path.resolve(path.dirname(astro.absolute), importedPath)) + return target + ? { target } + : { reason: `${component} selector "${selector}" target "${importedPath}" is missing or escapes the project` } +} + +export function readStaticExpression(node: unknown): { ok: true; value: unknown } | { ok: false; syntax: string } { + if (!node || typeof node !== "object") return { ok: false, syntax: "missing expression" } + const expression = node as AstRecord + const type = String(expression.type ?? "unknown") + + if (type === "Literal") return { ok: true, value: expression.value } + + if (type === "TemplateLiteral") { + const expressions = expression.expressions + const quasis = expression.quasis + if (!Array.isArray(expressions) || expressions.length !== 0 || !Array.isArray(quasis)) { + return { ok: false, syntax: type } + } + return { + ok: true, + value: quasis.map((quasi) => String(((quasi as AstRecord).value as AstRecord)?.cooked ?? "")).join(""), + } + } + + if (type === "ArrayExpression") { + if (!Array.isArray(expression.elements)) return { ok: false, syntax: type } + const values: unknown[] = [] + for (const element of expression.elements) { + if (!element || (element as AstRecord).type === "SpreadElement") return { ok: false, syntax: "SpreadElement" } + const result = readStaticExpression(element) + if (!result.ok) return result + values.push(result.value) + } + return { ok: true, value: values } + } + + if (type === "ObjectExpression") { + if (!Array.isArray(expression.properties)) return { ok: false, syntax: type } + const value: Record = {} + for (const propertyValue of expression.properties) { + const property = propertyValue as AstRecord + if (property.type !== "Property" || property.computed || property.kind !== "init") { + return { ok: false, syntax: String(property.type ?? "computed property") } + } + const keyNode = property.key as AstRecord + const key = + keyNode?.type === "Identifier" ? keyNode.name : keyNode?.type === "Literal" ? keyNode.value : undefined + if (typeof key !== "string" && typeof key !== "number") return { ok: false, syntax: "non-static key" } + const result = readStaticExpression(property.value) + if (!result.ok) return result + value[String(key)] = result.value + } + return { ok: true, value } + } + + return { ok: false, syntax: type } +} + +function staticAttribute(node: Node, name: string): { found: boolean; value?: unknown; syntax?: string } { + const attributes = (node as Node & { attributes?: AstRecord[] }).attributes ?? [] + const attribute = attributes.find( + (candidate) => candidate.type !== "mdxJsxExpressionAttribute" && candidate.name === name + ) + if (!attribute) return { found: false } + if (typeof attribute.value === "string" || attribute.value == null) + return { found: true, value: attribute.value ?? true } + const result = readStaticExpression(expressionFrom(attribute.value)) + return result.ok ? { found: true, value: result.value } : { found: true, syntax: result.syntax } +} + +function expressionAttributes(node: Node): Array<{ name: string; syntax: string }> { + const attributes = (node as Node & { attributes?: AstRecord[] }).attributes ?? [] + const failures: Array<{ name: string; syntax: string }> = [] + for (const attribute of attributes) { + if (attribute.type === "mdxJsxExpressionAttribute") { + failures.push({ name: "spread attribute", syntax: "mdxJsxExpressionAttribute" }) + continue + } + if (!attribute.value || typeof attribute.value === "string") continue + const result = readStaticExpression(expressionFrom(attribute.value)) + if (!result.ok) failures.push({ name: String(attribute.name ?? "attribute"), syntax: result.syntax }) + } + return failures +} + +function languageKeys(node: Node): { + keys: string[] + codes?: Array<{ key: string; value?: string; identifier?: string }> + syntax?: string +} { + const attributes = (node as Node & { attributes?: AstRecord[] }).attributes ?? [] + const attribute = attributes.find((candidate) => candidate.name === "languages") + const expression = expressionFrom(attribute?.value) + if (!expression || expression.type !== "ObjectExpression" || !Array.isArray(expression.properties)) { + return { keys: [], syntax: String(expression?.type ?? "missing languages") } + } + + const keys: string[] = [] + const codes: Array<{ key: string; value?: string; identifier?: string }> = [] + for (const propertyValue of expression.properties) { + const property = propertyValue as AstRecord + const keyNode = property.key as AstRecord + if (property.type !== "Property" || property.computed || property.kind !== "init") { + return { keys: [], syntax: String(property.type ?? "computed property") } + } + const key = keyNode?.type === "Identifier" ? keyNode.name : keyNode?.type === "Literal" ? keyNode.value : undefined + if (typeof key !== "string") return { keys: [], syntax: "non-static language key" } + + const value = property.value as AstRecord + const codeProperty = + value?.type === "ObjectExpression" && Array.isArray(value.properties) + ? (value.properties.find((candidate) => { + const item = candidate as AstRecord + const candidateKey = item.key as AstRecord + return ( + item.type === "Property" && + !item.computed && + item.kind === "init" && + (candidateKey?.name === "code" || candidateKey?.value === "code") + ) + }) as AstRecord | undefined) + : undefined + if (!codeProperty) return { keys: [], syntax: "non-static language branch" } + const codeNode = codeProperty.value as AstRecord + const staticCode = readStaticExpression(codeNode) + if (codeNode?.type === "Identifier" && typeof codeNode.name === "string") { + codes.push({ key, identifier: codeNode.name }) + } else if (staticCode.ok && typeof staticCode.value === "string") { + codes.push({ key, value: staticCode.value }) + } else { + return { keys: [], syntax: String(codeNode?.type ?? "non-static code") } + } + keys.push(key) + } + return { keys: [...new Set(keys)].sort(), codes } +} +type SchemaField = { field: string; type: string; description: string; link?: { label: string; href: string } } + +function extractStaticInitializer(source: string, declaration: string): string | null { + const declarationIndex = source.indexOf(declaration) + const equalsIndex = declarationIndex < 0 ? -1 : source.indexOf("=", declarationIndex + declaration.length) + const start = equalsIndex < 0 ? -1 : source.slice(equalsIndex + 1).search(/[[{]/) + equalsIndex + 1 + if (declarationIndex < 0 || equalsIndex < 0 || start <= equalsIndex) return null + const stack: string[] = [] + let quote = "" + let escaped = false + for (let index = start; index < source.length; index += 1) { + const character = source[index] + if (quote) { + if (escaped) escaped = false + else if (character === "\\") escaped = true + else if (character === quote) quote = "" + continue + } + if (character === '"' || character === "'" || character === "`") { + quote = character + continue + } + if (character === "{" || character === "[") stack.push(character) + if (character === "}" || character === "]") { + const expected = character === "}" ? "{" : "[" + if (stack.pop() !== expected) return null + if (stack.length === 0) return source.slice(start, index + 1) + } + } + return null +} + +function parseExpressionSource(source: string): AstRecord | null { + try { + const tree = processor.parse(`{${source}}`) + return expressionFrom(childrenOf(tree)[0]) + } catch { + return null + } +} + +function schemaFields(schema: string): SchemaField[] | null { + const dataFile = resolveProjectFile("src/features/feeds/components/reportSchemaData.ts") + if (!dataFile) return null + const source = fsSync.readFileSync(dataFile.absolute, "utf8") + const commonSource = extractStaticInitializer(source, "const COMMON_FIELDS") + const definitionsSource = extractStaticInitializer(source, "REPORT_SCHEMA_DEFINITIONS") + const common = commonSource ? readStaticExpression(parseExpressionSource(commonSource)) : null + const definitions = definitionsSource ? parseExpressionSource(definitionsSource) : null + if (!common?.ok || !Array.isArray(common.value) || definitions?.type !== "ObjectExpression") return null + const schemaProperty = Array.isArray(definitions.properties) + ? (definitions.properties.find((propertyValue) => { + const property = propertyValue as AstRecord + const key = property.key as AstRecord | undefined + return property.type === "Property" && !property.computed && (key?.name === schema || key?.value === schema) + }) as AstRecord | undefined) + : undefined + const schemaValue = schemaProperty?.value as AstRecord | undefined + const fieldsProperty = + schemaValue?.type === "ObjectExpression" && Array.isArray(schemaValue.properties) + ? (schemaValue.properties.find((propertyValue) => { + const property = propertyValue as AstRecord + const key = property.key as AstRecord | undefined + return ( + property.type === "Property" && !property.computed && (key?.name === "fields" || key?.value === "fields") + ) + }) as AstRecord | undefined) + : undefined + const fieldsExpression = fieldsProperty?.value as AstRecord | undefined + if (fieldsExpression?.type !== "ArrayExpression" || !Array.isArray(fieldsExpression.elements)) return null + const values: unknown[] = [] + for (const elementValue of fieldsExpression.elements) { + const element = elementValue as AstRecord + if (element?.type === "SpreadElement" && (element.argument as AstRecord | undefined)?.name === "COMMON_FIELDS") { + values.push(...common.value) + continue + } + const parsed = readStaticExpression(element) + if (!parsed.ok) return null + values.push(parsed.value) + } + if ( + !values.every( + (value): value is SchemaField => + !!value && + typeof value === "object" && + typeof (value as SchemaField).field === "string" && + typeof (value as SchemaField).type === "string" && + typeof (value as SchemaField).description === "string" && + (!(value as SchemaField).link || + (typeof (value as SchemaField).link?.label === "string" && + typeof (value as SchemaField).link?.href === "string")) + ) + ) { + return null + } + return values +} + +export function analyzeSourceMarkdown( + source: string, + sourcePath?: string, + ancestorPaths: ReadonlySet = new Set() +): SourceAnalysis { + let tree: Node + try { + tree = processor.parse(maskFrontmatter(source)) + } catch (error) { + return { + facts: [], + diagnostics: [ + { + status: "unverifiable", + ordinal: 1, + name: "MDX parse error", + line: 1, + sourceText: source.split(/\r?\n/, 1)[0] ?? "", + reason: error instanceof Error ? error.message : "Raw source could not be parsed", + ...(sourcePath ? { sourcePath } : {}), + }, + ], + languages: [], + } + } + const imports = staticImports(tree) + const lines = source.split(/\r?\n/) + const facts: GroupedFact[] = [] + const diagnostics: SourceDiagnostic[] = [] + const languages = new Set() + const parentByNode = new WeakMap() + const groupByBlock = new WeakMap() + const segmentByBlock = new WeakMap() + let groupOrdinal = 0 + let factOrdinal = 0 + let diagnosticOrdinal = 0 + + const inlineBlock = (node: Node): Node | null => { + let current: Node | undefined = node + while (current) { + if ( + current.type === "paragraph" || + current.type === "tableCell" || + current.type === "mdxJsxFlowElement" || + current.type === "mdxJsxTextElement" + ) { + return current + } + current = parentByNode.get(current) + } + return null + } + + const textGroup = (node: Node): string | undefined => { + const block = inlineBlock(node) + if (!block) return undefined + let group = groupByBlock.get(block) + if (group === undefined) { + group = ++groupOrdinal + groupByBlock.set(block, group) + } + return `${group}:${segmentByBlock.get(block) ?? 0}` + } + + const breakTextGroup = (node: Node) => { + const ownBlock = inlineBlock(node) + const block = ownBlock === node ? inlineBlock(parentByNode.get(node) ?? node) : ownBlock + if (block) segmentByBlock.set(block, (segmentByBlock.get(block) ?? 0) + 1) + } + + const addFact = ( + kind: SourceFact["kind"], + value: string, + node: Node, + url?: string, + depth?: number, + variant?: string, + coalesce = false + ) => { + const normalized = normalizeText(value) + if (!normalized) { + if (coalesce) { + const group = textGroup(node) + const previous = facts[facts.length - 1] + if (group && previous?.kind === "text" && previous.group === group) { + previous.rawValue = `${previous.rawValue ?? previous.value}${value}` + } + } + return + } + const line = nodeLine(node) + facts.push({ + ordinal: ++factOrdinal, + kind, + value: normalized, + url, + depth, + variant, + line, + sourceText: lineText(lines, line), + ...(sourcePath ? { sourcePath } : {}), + ...(coalesce ? { group: textGroup(node), rawValue: value } : {}), + }) + } + const addDiagnostic = (status: SourceDiagnostic["status"], name: string, node: Node, reason: string) => { + const line = nodeLine(node) + diagnostics.push({ + status, + ordinal: ++diagnosticOrdinal, + name, + line, + sourceText: lineText(lines, line), + reason, + ...(sourcePath ? { sourcePath } : {}), + }) + } + const appendAnalysis = (analysis: SourceAnalysis) => { + for (const fact of analysis.facts) facts.push({ ...fact, ordinal: ++factOrdinal }) + for (const diagnostic of analysis.diagnostics) diagnostics.push({ ...diagnostic, ordinal: ++diagnosticOrdinal }) + analysis.languages.forEach((language) => languages.add(language)) + } + + const includeMarkdown = (component: string, node: Node, target: { absolute: string; relative: string }) => { + if (ancestorPaths.has(target.absolute)) { + addDiagnostic( + "unverifiable", + component, + node, + `${component} target ${target.relative} forms a static inclusion cycle` + ) + return + } + try { + const nestedAncestors = new Set(ancestorPaths) + nestedAncestors.add(target.absolute) + appendAnalysis( + analyzeSourceMarkdown(fsSync.readFileSync(target.absolute, "utf8"), target.relative, nestedAncestors) + ) + } catch (error) { + addDiagnostic( + "unverifiable", + component, + node, + `${component} target ${target.relative} could not be read: ${error instanceof Error ? error.message : "unknown error"}` + ) + } + } + + const inspect = (root: Node) => { + visit(root, (node, _index, parent) => { + if (parent && !parentByNode.has(node)) parentByNode.set(node, parent) + }) + visit(root, (node) => { + if (node.type === "heading") { + const depth = "depth" in node && typeof node.depth === "number" ? node.depth : undefined + addFact("heading", nodeVisibleText(node), node, undefined, depth) + visit(node, "link", (link) => { + addFact("link", nodeVisibleText(link), link, String((link as Node & { url?: unknown }).url ?? "")) + return SKIP + }) + return SKIP + } + if (node.type === "link") { + addFact("link", nodeVisibleText(node), node, String((node as Node & { url?: unknown }).url ?? "")) + return SKIP + } + if (node.type === "image") { + const alt = String((node as Node & { alt?: unknown }).alt ?? "Image") || "Image" + addFact("text", `(Image: ${alt})`, node) + return SKIP + } + if (node.type === "code") { + addFact("code", String((node as Node & { value?: unknown }).value ?? ""), node) + return SKIP + } + if (node.type === "inlineCode" || node.type === "text") { + addFact( + "text", + String((node as Node & { value?: unknown }).value ?? ""), + node, + undefined, + undefined, + undefined, + true + ) + return + } + if (node.type === "html") { + breakTextGroup(node) + addDiagnostic("unverifiable", "HTML", node, "Raw HTML is not statically projected") + return SKIP + } + if (node.type === "mdxFlowExpression" || node.type === "mdxTextExpression") { + const raw = String((node as Node & { value?: unknown }).value ?? "").trim() + if (!raw || /^\/\*[\s\S]*\*\/$/.test(raw)) return SKIP + const result = readStaticExpression(expressionFrom(node)) + if (result.ok && (typeof result.value === "string" || typeof result.value === "number")) { + addFact("text", String(result.value), node, undefined, undefined, undefined, true) + } else { + breakTextGroup(node) + addDiagnostic( + "unverifiable", + raw, + node, + `Dynamic MDX expression (${result.ok ? "non-text value" : result.syntax})` + ) + } + return SKIP + } + if (node.type !== "mdxJsxFlowElement" && node.type !== "mdxJsxTextElement") return + + const name = String((node as Node & { name?: unknown }).name ?? "") + if (!name) { + inspect({ type: "root", children: childrenOf(node) } as Parent) + return SKIP + } + if (containerElements[name]) { + for (const failure of expressionAttributes(node)) { + addDiagnostic("unverifiable", `${name}.${failure.name}`, node, `Dynamic JSX attribute (${failure.syntax})`) + } + inspect({ type: "root", children: childrenOf(node) } as Parent) + return SKIP + } + if (/^[a-z]/.test(name)) { + breakTextGroup(node) + addDiagnostic("unverifiable", name, node, `Raw HTML element ${name} is not statically projected`) + inspect({ type: "root", children: childrenOf(node) } as Parent) + breakTextGroup(node) + return SKIP + } + + if (name === "Aside" || name === "Callout") { + for (const failure of expressionAttributes(node)) { + addDiagnostic("unverifiable", `${name}.${failure.name}`, node, `Dynamic JSX attribute (${failure.syntax})`) + } + const type = staticAttribute(node, "type") + const title = staticAttribute(node, "title") + if (!type.syntax && !title.syntax) { + const typeText = typeof type.value === "string" ? type.value.toUpperCase() : "NOTE" + const titleText = typeof title.value === "string" && title.value ? `: ${title.value}` : "" + addFact("text", `${typeText}${titleText}`, node) + } + inspect({ type: "root", children: childrenOf(node) } as Parent) + return SKIP + } + + if (name === "CopyText") { + const text = staticAttribute(node, "text") + if (text.syntax || !text.found || typeof text.value !== "string") { + addDiagnostic("unverifiable", "CopyText.text", node, `Dynamic CopyText text (${text.syntax ?? "missing"})`) + } else { + addFact("text", text.value, node) + } + return SKIP + } + + if (name === "ClickToZoom") { + const src = staticAttribute(node, "src") + const alt = staticAttribute(node, "alt") + if (src.syntax || !src.found || typeof src.value !== "string" || alt.syntax) { + addDiagnostic( + "unverifiable", + "ClickToZoom", + node, + `Dynamic image attributes (${src.syntax ?? alt.syntax ?? "missing src"})` + ) + } else { + addFact("text", `(Image: ${typeof alt.value === "string" && alt.value ? alt.value : "Image"})`, node) + } + return SKIP + } + + if (name === "Address") { + const contractUrl = staticAttribute(node, "contractUrl") + const address = staticAttribute(node, "address") + const endLength = staticAttribute(node, "endLength") + if ( + contractUrl.syntax || + typeof contractUrl.value !== "string" || + address.syntax || + (address.found && typeof address.value !== "string") || + endLength.syntax || + (endLength.found && + (typeof endLength.value !== "number" || !Number.isInteger(endLength.value) || endLength.value < 0)) + ) { + addDiagnostic( + "unverifiable", + "Address", + node, + `Dynamic address attributes (${contractUrl.syntax ?? address.syntax ?? endLength.syntax ?? "missing contractUrl"})` + ) + } else { + const urlTail = contractUrl.value.split("/").pop() ?? contractUrl.value + const fullDisplay = typeof address.value === "string" ? address.value : urlTail + const display = + typeof endLength.value === "number" && endLength.value > 0 + ? `${fullDisplay.slice(0, endLength.value + 2)}...${fullDisplay.slice(-endLength.value)}` + : fullDisplay + addFact("link", display, node, contractUrl.value) + } + return SKIP + } + + if (name === "CodeHighlightBlock") { + const code = staticAttribute(node, "code") + const title = staticAttribute(node, "title") + if (title.syntax || (title.found && typeof title.value !== "string")) { + addDiagnostic( + "unverifiable", + "CodeHighlightBlock", + node, + `Dynamic title (${title.syntax ?? "invalid static type"})` + ) + } else if (typeof title.value === "string" && title.value) { + addFact("text", `Code snippet for ${title.value}:`, node) + } + if (typeof code.value === "string") { + addFact("code", code.value, node) + return SKIP + } + const expression = expressionAttribute(node, "code") + const identifier = + expression?.type === "Identifier" && typeof expression.name === "string" ? expression.name : undefined + const importedPath = identifier ? imports.get(identifier) : undefined + const location = sourceLocation(sourcePath) + const target = + importedPath && location + ? resolveProjectFile(path.resolve(path.dirname(location.absolute), importedPath.split("?")[0])) + : null + if (!target) { + addDiagnostic( + "unverifiable", + "CodeHighlightBlock", + node, + `CodeHighlightBlock code target "${importedPath ?? code.syntax ?? "missing"}" could not be statically resolved` + ) + } else { + addFact("code", stripHighlighterComments(fsSync.readFileSync(target.absolute, "utf8")), node) + } + return SKIP + } + + if (name === "CodeSample") { + const src = staticAttribute(node, "src") + const showButtonOnly = staticAttribute(node, "showButtonOnly") + if ( + typeof src.value !== "string" || + !src.value || + showButtonOnly.syntax || + (showButtonOnly.found && typeof showButtonOnly.value !== "boolean") + ) { + addDiagnostic( + "unverifiable", + "CodeSample", + node, + `CodeSample path "${typeof src.value === "string" ? src.value : (src.syntax ?? "missing")}" is not statically resolvable` + ) + return SKIP + } + if (showButtonOnly.value === true) { + addFact( + "link", + `Open ${path.basename(src.value)} in Remix`, + node, + `https://remix.ethereum.org/#url=https://docs.chain.link/${src.value}` + ) + return SKIP + } + const target = [ + path.resolve("public", src.value), + path.resolve(src.value), + path.resolve("src", src.value), + ].reduce>( + (found, candidate) => found ?? resolveProjectFile(candidate), + null + ) + if (!target) { + addDiagnostic( + "unverifiable", + "CodeSample", + node, + `CodeSample path "${src.value}" is missing or escapes the project` + ) + } else { + addFact("code", fsSync.readFileSync(target.absolute, "utf8"), node) + } + return SKIP + } + + if (name in selectorComponents) { + const component = name as keyof typeof selectorComponents + const selector = staticAttribute(node, selectorComponents[component].attribute) + if (typeof selector.value !== "string" || !selector.value) { + addDiagnostic( + "unverifiable", + component, + node, + `${component} selector is dynamic or missing (${selector.syntax ?? "missing"})` + ) + return SKIP + } + const resolution = resolveSelectorTarget(component, selector.value) + if (!resolution.target) { + addDiagnostic( + "unverifiable", + component, + node, + resolution.reason ?? `${component} target could not be resolved` + ) + } else { + includeMarkdown(component, node, resolution.target) + } + return SKIP + } + + if (name === "SchemaFieldsTable") { + const schema = staticAttribute(node, "schema") + const fields = typeof schema.value === "string" ? schemaFields(schema.value) : null + if (!fields) { + addDiagnostic( + "unverifiable", + "SchemaFieldsTable", + node, + typeof schema.value === "string" + ? `SchemaFieldsTable schema "${schema.value}" could not be read from static schema definitions` + : `SchemaFieldsTable schema is dynamic or missing (${schema.syntax ?? "missing"})` + ) + return SKIP + } + addFact("text", "Field", node) + addFact("text", "Type", node) + addFact("text", "Description", node) + for (const field of fields) { + addFact("text", field.field, node) + addFact("text", field.type, node) + addFact("text", field.link ? `${field.description} —` : field.description, node) + if (field.link) addFact("link", field.link.label, node, field.link.href) + } + return SKIP + } + if (name === "Billing") { + addDiagnostic( + "unverifiable", + "Billing", + node, + "Billing content depends on imported fee configuration and runtime calculations" + ) + return SKIP + } + + if (name === "PageTabs") { + const pages = staticAttribute(node, "pages") + const title = staticAttribute(node, "headerTitle") + const description = staticAttribute(node, "headerDescription") + const showHeader = staticAttribute(node, "showHeader") + if (pages.syntax || !pages.found || !Array.isArray(pages.value)) { + addDiagnostic("unverifiable", "PageTabs.pages", node, `Dynamic PageTabs pages (${pages.syntax ?? "missing"})`) + return SKIP + } + if ( + title.syntax || + (title.found && typeof title.value !== "string") || + description.syntax || + (description.found && typeof description.value !== "string") || + showHeader.syntax || + (showHeader.found && typeof showHeader.value !== "boolean") + ) { + addDiagnostic( + "unverifiable", + "PageTabs.header", + node, + `Dynamic PageTabs header (${title.syntax ?? description.syntax ?? showHeader.syntax ?? "invalid static type"})` + ) + return SKIP + } + if (showHeader.value !== false) { + if (!title.found || title.value === true) { + addFact("heading", "Guide Versions", node, undefined, 2) + } else if (typeof title.value === "string" && title.value) { + addFact("heading", title.value, node, undefined, 2) + } + if (typeof description.value === "string" && description.value) addFact("text", description.value, node) + } + for (const entry of pages.value) { + const group = Array.isArray(entry) ? entry : [entry] + if (!group.length || group.some((item) => !item || typeof item !== "object")) { + addDiagnostic("unverifiable", "PageTabs.pages", node, "PageTabs contains a non-static group") + continue + } + const records = group as Record[] + const labels = records.map((item) => item.name).filter((value): value is string => typeof value === "string") + const firstUrl = records[0].url + if (labels.length !== records.length || typeof firstUrl !== "string") { + addDiagnostic("unverifiable", "PageTabs.pages", node, "PageTabs group requires static name and URL values") + continue + } + addFact("link", labels.join(" / "), node, firstUrl) + } + return SKIP + } + + if (name === "Tabs" || name === "TabsContent") { + const tabs: Array<{ key: string; node: Node }> = [] + const panels: Record = {} + for (const child of childrenOf(node)) { + const slot = staticAttribute(child, "slot") + if (slot.syntax || typeof slot.value !== "string") { + addDiagnostic("unverifiable", "Tabs.slot", child, `Dynamic tab slot (${slot.syntax ?? "missing"})`) + continue + } + if (slot.value.startsWith("tab.")) tabs.push({ key: slot.value.slice(4), node: child }) + if (slot.value.startsWith("panel.")) panels[slot.value.slice(6)] = child + } + for (const tab of tabs) { + addFact("heading", nodeVisibleText(tab.node), tab.node, undefined, 3) + const panel = panels[tab.key] + if (panel) { + inspect({ type: "root", children: childrenOf(panel) } as Parent) + } else { + addDiagnostic("unverifiable", `Tabs.panel.${tab.key}`, tab.node, "Tab has no matching static panel") + } + } + return SKIP + } + + if (name === "PackageManagerTabs") { + const slots: Record = {} + for (const child of childrenOf(node)) { + const slot = staticAttribute(child, "slot") + if (slot.syntax || typeof slot.value !== "string") { + addDiagnostic( + "unverifiable", + "PackageManagerTabs.slot", + child, + `Dynamic package slot (${slot.syntax ?? "missing"})` + ) + } else { + slots[slot.value] = child + } + } + for (const packageManager of ["npm", "yarn"]) { + const panel = slots[packageManager] + if (!panel) continue + addFact("heading", packageManager, panel, undefined, 3) + inspect({ type: "root", children: childrenOf(panel) } as Parent) + } + return SKIP + } + + if (name === "Accordion") { + const title = staticAttribute(node, "title") + const number = staticAttribute(node, "number") + if ( + title.syntax || + typeof title.value !== "string" || + number.syntax || + (number.found && typeof number.value !== "number") + ) { + addDiagnostic( + "unverifiable", + "Accordion", + node, + `Dynamic accordion heading (${title.syntax ?? number.syntax ?? "missing title"})` + ) + } else { + const prefix = number.found ? `${number.value}. ` : "" + addFact("heading", `${prefix}${title.value}`, node, undefined, 3) + } + inspect({ type: "root", children: childrenOf(node) } as Parent) + return SKIP + } + + if (name === "CodeHighlightBlockMulti") { + const result = languageKeys(node) + if (result.syntax) { + addDiagnostic( + "unverifiable", + "CodeHighlightBlockMulti.languages", + node, + `Dynamic languages (${result.syntax})` + ) + } else { + result.keys.forEach((key) => languages.add(key)) + for (const code of result.codes ?? []) { + if (code.value !== undefined) { + addFact("code", code.value, node, undefined, undefined, code.key) + continue + } + const importedPath = code.identifier ? imports.get(code.identifier) : undefined + const location = sourceLocation(sourcePath) + const target = + importedPath && location + ? resolveProjectFile(path.resolve(path.dirname(location.absolute), importedPath.split("?")[0])) + : null + if (!target) { + addDiagnostic( + "unverifiable", + `CodeHighlightBlockMulti.languages.${code.key}`, + node, + `Imported code identifier "${code.identifier ?? "missing"}" could not be resolved through a contained static import` + ) + continue + } + try { + addFact( + "code", + stripHighlighterComments(fsSync.readFileSync(target.absolute, "utf8")), + node, + undefined, + undefined, + code.key + ) + } catch (error) { + addDiagnostic( + "unverifiable", + `CodeHighlightBlockMulti.languages.${code.key}`, + node, + `Imported code target ${target.relative} could not be read: ${ + error instanceof Error ? error.message : "unknown error" + }` + ) + } + } + } + return SKIP + } + + addDiagnostic("unsupported", name, node, `Unsupported MDX component ${name}`) + return SKIP + }) + } + + inspect(tree) + return { facts: coalesceTextFacts(facts), diagnostics, languages: [...languages].sort() } +} + +function analyzeObservedMarkdown(markdown: string): ObservedAnalysis { + let tree: Node + try { + tree = processor.parse(markdown) + } catch (error) { + return { + facts: [], + residuals: [ + { + name: "Markdown parse error", + line: 1, + text: markdown, + reason: error instanceof Error ? error.message : "Served Markdown could not be parsed", + }, + ], + } + } + const facts: GroupedFact[] = [] + const residuals: ObservedAnalysis["residuals"] = [] + const parentByNode = new WeakMap() + const groupByBlock = new WeakMap() + const segmentByBlock = new WeakMap() + let groupOrdinal = 0 + + visit(tree, (node, _index, parent) => { + if (parent) parentByNode.set(node, parent) + }) + + const inlineBlock = (node: Node): Node | null => { + let current: Node | undefined = node + while (current) { + if ( + current.type === "paragraph" || + current.type === "tableCell" || + current.type === "mdxJsxFlowElement" || + current.type === "mdxJsxTextElement" + ) { + return current + } + current = parentByNode.get(current) + } + return null + } + + const textGroup = (node: Node): string | undefined => { + const block = inlineBlock(node) + if (!block) return undefined + let group = groupByBlock.get(block) + if (group === undefined) { + group = ++groupOrdinal + groupByBlock.set(block, group) + } + return `${group}:${segmentByBlock.get(block) ?? 0}` + } + + const breakTextGroup = (node: Node) => { + const ownBlock = inlineBlock(node) + const block = ownBlock === node ? inlineBlock(parentByNode.get(node) ?? node) : ownBlock + if (block) segmentByBlock.set(block, (segmentByBlock.get(block) ?? 0) + 1) + } + + const addFact = (fact: ObservedFact, node: Node, rawValue?: string) => { + const group = rawValue === undefined ? undefined : textGroup(node) + if (fact.kind === "text" && !fact.value && group) { + const previous = facts[facts.length - 1] + if (previous?.kind === "text" && previous.group === group) { + previous.rawValue = `${previous.rawValue ?? previous.value}${rawValue}` + } + return + } + facts.push({ ...fact, ...(rawValue === undefined ? {} : { group, rawValue }) }) + } + + const exactText = (node: Node): string => { + const start = node.position?.start.offset + const end = node.position?.end.offset + return typeof start === "number" && typeof end === "number" + ? markdown.slice(start, end) + : lineText(markdown.split(/\r?\n/), nodeLine(node)) + } + + visit(tree, (node) => { + if (node.type === "heading") { + const value = normalizeText(nodeVisibleText(node)) + const depth = "depth" in node && typeof node.depth === "number" ? node.depth : undefined + if (value) addFact({ kind: "heading", value, depth }, node) + visit(node, "link", (link) => { + const label = normalizeText(nodeVisibleText(link)) + if (label) { + addFact({ kind: "link", value: label, url: String((link as Node & { url?: unknown }).url ?? "") }, link) + } + return SKIP + }) + return SKIP + } + if (node.type === "link") { + const value = normalizeText(nodeVisibleText(node)) + if (value) addFact({ kind: "link", value, url: String((node as Node & { url?: unknown }).url ?? "") }, node) + return SKIP + } + if (node.type === "image") { + const alt = String((node as Node & { alt?: unknown }).alt ?? "Image") || "Image" + addFact({ kind: "text", value: `(Image: ${alt})` }, node) + return SKIP + } + if (node.type === "code") { + const value = normalizeText(String((node as Node & { value?: unknown }).value ?? "")) + if (value) addFact({ kind: "code", value }, node) + return SKIP + } + if (node.type === "inlineCode" || node.type === "text") { + const raw = String((node as Node & { value?: unknown }).value ?? "") + const value = normalizeText(raw) + addFact({ kind: "text", value }, node, raw) + return + } + if ( + node.type === "html" || + node.type === "mdxJsxFlowElement" || + node.type === "mdxJsxTextElement" || + node.type === "mdxFlowExpression" || + node.type === "mdxTextExpression" || + node.type === "mdxjsEsm" + ) { + breakTextGroup(node) + residuals.push({ + name: + node.type === "html" + ? "HTML" + : node.type === "mdxJsxFlowElement" || node.type === "mdxJsxTextElement" + ? String((node as Node & { name?: unknown }).name ?? "MDX") + : node.type, + line: nodeLine(node), + text: exactText(node), + reason: "Served Markdown contains residual runtime syntax", + }) + return SKIP + } + }) + + return { facts: coalesceTextFacts(facts), residuals } +} + +function factMatches(source: SourceFact, observed: ObservedFact): boolean { + return ( + source.kind === observed.kind && + source.value === observed.value && + (source.kind !== "heading" || source.depth === observed.depth) && + (source.kind !== "link" || source.url === observed.url) + ) +} +function frontmatterTitle(source: string): { title?: string; line: number } { + if (!source.startsWith("---")) return { line: 1 } + const end = source.indexOf("\n---", 3) + if (end < 0) return { line: 1 } + const frontmatter = source.slice(3, end) + const match = /^\s*title:\s*"?(.+?)"?\s*$/m.exec(frontmatter) + if (!match) return { line: 1 } + const line = source.slice(0, 3 + (match.index ?? 0)).split(/\r?\n/).length + return { title: match[1], line } +} + +function expectedCanonicalSource( + requestPath: string, + sourcePath: string, + routeKind: MarkdownArtifact["routeKind"] +): string { + if (routeKind === "special") return `${SITE_BASE}/${requestPath}` + const relative = sourcePath + .split(path.sep) + .join("/") + .replace(/^.*?src\/content\//, "") + const section = requestPath.split("/")[0] + let slug = sourceRoute(relative) + if (!slug.startsWith(section)) slug = `${section}/${slug}` + return `${SITE_BASE}/${slug}` +} + +function inspectNormalEnvelope( + requestPath: string, + sourcePath: string, + source: string, + artifact: MarkdownArtifact, + servedLines: string[], + lang: string, + exceptions: readonly FidelityException[] +): FidelityFinding[] { + const frontmatter = frontmatterTitle(source) + const expectedTitle = frontmatter.title || path.basename(sourcePath, path.extname(sourcePath)) + const expectedSource = expectedCanonicalSource(requestPath, sourcePath, artifact.routeKind) + const directiveLine = servedLines[2]?.startsWith("Last Updated: ") ? 4 : 3 + const fields = [ + { name: "title", expected: `# ${expectedTitle}`, actual: servedLines[0], sourceLine: frontmatter.line }, + { name: "source", expected: `Source: ${expectedSource}`, actual: servedLines[1], sourceLine: null }, + { name: "directive", expected: LLMS_DIRECTIVE, actual: servedLines[directiveLine], sourceLine: null }, + ] as const + + return fields.map((field) => { + const present = field.actual === field.expected + return withException( + { + path: requestPath, + status: present ? "present" : "missing", + occurrence: `lang=${lang};envelope=${JSON.stringify({ field: field.name, expected: field.expected })};duplicate=1`, + sourcePath, + sourceLine: field.sourceLine, + ...(field.sourceLine === null ? {} : { sourceText: lineText(source.split(/\r?\n/), field.sourceLine) }), + ...(lang === "default" ? {} : { lang }), + name: `Envelope.${field.name}`, + expected: field.expected, + ...(present ? {} : { reason: `Normal artifact envelope ${field.name} is missing or changed` }), + display: shortValue(field.expected), + }, + exceptions + ) + }) +} + +function shortValue(value: string): string { + return value.length <= 80 ? value : `${value.slice(0, 77)}...` +} + +function factSemantic(fact: SourceFact): string { + return JSON.stringify({ + kind: fact.kind, + value: fact.value, + ...(fact.url === undefined ? {} : { url: fact.url }), + ...(fact.depth === undefined ? {} : { depth: fact.depth }), + }) +} + +function occurrenceForFact(fact: SourceFact, lang: string, duplicate: number): string { + return `lang=${lang};fact=${factSemantic(fact)};duplicate=${duplicate}` +} + +function diagnosticSemantic(diagnostic: Pick): string { + return JSON.stringify({ component: diagnostic.name, reason: diagnostic.reason }) +} + +function occurrenceForDiagnostic(diagnostic: SourceDiagnostic, lang: string, duplicate: number): string { + return `lang=${lang};diagnostic=${diagnosticSemantic(diagnostic)};duplicate=${duplicate}` +} + +function residualSemantic(residual: ObservedAnalysis["residuals"][number]): string { + return JSON.stringify({ component: residual.name, reason: residual.reason, servedText: residual.text }) +} + +export function withException(finding: FidelityFinding, exceptions: readonly FidelityException[]): FidelityFinding { + if (finding.status === "present") return finding + const exception = exceptions.find( + (candidate) => + candidate.path === finding.path && + candidate.occurrence === finding.occurrence && + candidate.status === finding.status && + candidate.reason.trim().length > 0 && + candidate.owner.trim().length > 0 && + candidate.removalCondition.trim().length > 0 + ) + return exception + ? { + ...finding, + exception: { + reason: exception.reason, + owner: exception.owner, + removalCondition: exception.removalCondition, + }, + } + : finding +} + +export function compareSourceToArtifact( + requestPath: string, + sourcePath: string, + source: string, + artifact: MarkdownArtifact, + lang = "default", + exceptions: readonly FidelityException[] = markdownFidelityExceptions +): FidelityFinding[] { + const analysis = analyzeSourceMarkdown(source, sourcePath) + const servedLines = artifact.markdown.split(/\r?\n/) + const directiveIndex = servedLines.findIndex((line) => line === LLMS_DIRECTIVE) + const servedBody = directiveIndex >= 0 ? servedLines.slice(directiveIndex + 1).join("\n") : artifact.markdown + const servedLineOffset = directiveIndex >= 0 ? directiveIndex + 1 : 0 + const observed = analyzeObservedMarkdown(servedBody) + const findings: FidelityFinding[] = + artifact.sourcePath && !path.isAbsolute(artifact.sourcePath) + ? inspectNormalEnvelope(requestPath, sourcePath, source, artifact, servedLines, lang, exceptions) + : [] + let observedIndex = 0 + const presentFactDuplicates = new Map() + const missingFactDuplicates = new Map() + const diagnosticDuplicates = new Map() + const residualDuplicates = new Map() + + for (const fact of analysis.facts.filter( + (candidate) => lang === "default" || !candidate.variant || candidate.variant === lang + )) { + const semantic = factSemantic(fact) + let matchedAt = -1 + for (let index = observedIndex; index < observed.facts.length; index += 1) { + if (factMatches(fact, observed.facts[index])) { + matchedAt = index + break + } + } + const duplicateMap = matchedAt >= 0 ? presentFactDuplicates : missingFactDuplicates + const duplicate = (duplicateMap.get(semantic) ?? 0) + 1 + duplicateMap.set(semantic, duplicate) + const finding: FidelityFinding = { + ...(matchedAt >= 0 ? {} : { reason: "Expected source fact is missing from served Markdown" }), + path: requestPath, + status: matchedAt >= 0 ? "present" : "missing", + occurrence: occurrenceForFact(fact, lang, duplicate), + sourcePath: fact.sourcePath ?? sourcePath, + sourceLine: fact.line, + sourceText: fact.sourceText, + ...(lang === "default" ? {} : { lang }), + expected: fact.kind === "link" ? `${fact.value} -> ${fact.url}` : fact.value, + display: shortValue(fact.kind === "link" ? `${fact.value} -> ${fact.url}` : fact.value), + } + findings.push(withException(finding, exceptions)) + if (matchedAt >= 0) observedIndex = matchedAt + 1 + } + + for (const diagnostic of analysis.diagnostics) { + const semantic = diagnosticSemantic(diagnostic) + const duplicate = (diagnosticDuplicates.get(semantic) ?? 0) + 1 + diagnosticDuplicates.set(semantic, duplicate) + findings.push( + withException( + { + path: requestPath, + status: diagnostic.status, + occurrence: occurrenceForDiagnostic(diagnostic, lang, duplicate), + sourcePath: diagnostic.sourcePath ?? sourcePath, + sourceLine: diagnostic.line, + sourceText: diagnostic.sourceText, + ...(lang === "default" ? {} : { lang }), + name: diagnostic.name, + reason: diagnostic.reason, + }, + exceptions + ) + ) + } + + observed.residuals.forEach((residual) => { + const semantic = residualSemantic(residual) + const duplicate = (residualDuplicates.get(semantic) ?? 0) + 1 + residualDuplicates.set(semantic, duplicate) + findings.push( + withException( + { + path: requestPath, + status: "unverifiable", + occurrence: `lang=${lang};residual=${semantic};duplicate=${duplicate}`, + sourcePath, + sourceLine: null, + ...(lang === "default" ? {} : { lang }), + name: residual.name, + reason: residual.reason, + servedLine: residual.line + servedLineOffset, + servedText: residual.text, + display: shortValue(residual.text), + }, + exceptions + ) + ) + }) + + return findings +} + +export function findingIdentity(finding: FidelityFinding): string { + return JSON.stringify({ + path: finding.path, + status: finding.status, + language: finding.lang ?? "default", + occurrence: finding.occurrence, + ...(finding.name === undefined ? {} : { component: finding.name }), + ...(finding.expected === undefined ? {} : { expected: finding.expected }), + ...(finding.reason === undefined ? {} : { reason: finding.reason }), + ...(finding.servedText === undefined ? {} : { servedText: finding.servedText }), + }) +} + +export function determineExitCode(mode: RunMode, findings: readonly FidelityFinding[]): 0 | 1 { + return mode === "focused" && findings.some((finding) => finding.status !== "present" && !finding.exception) ? 1 : 0 +} + +function compareFinding(left: FidelityFinding, right: FidelityFinding): number { + const leftIdentity = findingIdentity(left) + const rightIdentity = findingIdentity(right) + return leftIdentity < rightIdentity ? -1 : leftIdentity > rightIdentity ? 1 : 0 +} + +export function createReport(pathCount: number, findings: readonly FidelityFinding[]): FidelityReport { + const counts: Record = { + present: 0, + missing: 0, + unsupported: 0, + unverifiable: 0, + degraded: 0, + } + findings.forEach((finding) => { + counts[finding.status] += 1 + }) + return { pathCount, counts, findings: [...findings].sort(compareFinding) } +} + +export function serializeReport(report: FidelityReport): string { + return `${JSON.stringify(report, null, 2)}\n` +} + +function cliRequestPath(value: string): string | null { + if (path.isAbsolute(value) || value.includes("\\")) return null + const segments = value.split("/") + if (segments.some((segment) => !segment || segment === "." || segment === "..")) return null + + if (value.startsWith("src/content/")) { + if (!/\.(?:md|mdx)$/i.test(value)) return null + const withoutExtension = value.replace(/\.(?:md|mdx)$/i, "") + if (/\.(?:md|mdx)$/i.test(withoutExtension)) return null + const relativePath = value.slice("src/content/".length) + return normalizeMarkdownPath(sourceRoute(relativePath)) + } + + if (value === "src/content" || value.startsWith("src/") || /\.(?:md|mdx)$/i.test(value)) return null + return normalizeMarkdownPath(value) +} + +export function parseCliArguments(argv: readonly string[]): { mode: RunMode; paths: string[] } { + const paths: string[] = [] + for (let index = 0; index < argv.length; index += 1) { + if (argv[index] !== "--path") throw new Error(`Unknown argument: ${argv[index]}`) + const value = argv[index + 1] + if (!value || value.startsWith("--")) throw new Error("--path requires a value") + const normalized = cliRequestPath(value) + if (!normalized) throw new Error(`Invalid Markdown path: ${value}`) + paths.push(normalized) + index += 1 + } + return paths.length ? { mode: "focused", paths: [...new Set(paths)].sort() } : { mode: "full-corpus", paths: [] } +} + +function sourceRoute(relativePath: string): string { + const withoutExtension = relativePath + .replace(/\.(?:md|mdx)$/i, "") + .split(path.sep) + .join("/") + return withoutExtension.endsWith("/index") ? withoutExtension.slice(0, -"/index".length) : withoutExtension +} + +export async function collectCorpusPaths(contentRoot = CONTENT_ROOT): Promise { + const routes = new Set([...MARKDOWN_REDIRECT_PATHS, "cre-templates"]) + + const walk = async (directory: string): Promise => { + const entries = await fs.readdir(directory, { withFileTypes: true }) + entries.sort((left, right) => left.name.localeCompare(right.name)) + for (const entry of entries) { + const absolute = path.join(directory, entry.name) + if (entry.isDirectory()) { + await walk(absolute) + } else if (/\.(?:md|mdx)$/i.test(entry.name) && !/^llms-full/i.test(entry.name)) { + routes.add(sourceRoute(path.relative(contentRoot, absolute))) + } + } + } + + await walk(contentRoot) + for (const route of [...routes]) { + if (route.startsWith("cre/") && (route.endsWith("-go") || route.endsWith("-ts"))) { + routes.add(route.slice(0, -3)) + } + } + return [...routes].sort() +} + +function safeSourcePath(sourcePath: string): { absolute: string; relative: string } | null { + const absolute = path.isAbsolute(sourcePath) ? path.resolve(sourcePath) : path.resolve(CONTENT_ROOT, sourcePath) + if (absolute !== CONTENT_ROOT && !absolute.startsWith(`${CONTENT_ROOT}${path.sep}`)) return null + return { absolute, relative: path.relative(process.cwd(), absolute).split(path.sep).join("/") } +} + +export function inspectSyntheticArtifact( + requestPath: string, + artifact: MarkdownArtifact, + exceptions: readonly FidelityException[] = markdownFidelityExceptions +): { findings: FidelityFinding[]; targetPaths: string[] } { + const finding = ( + status: "present" | "missing" | "unverifiable", + occurrence: string, + expected: string, + reason?: string + ) => + withException( + { + path: requestPath, + status, + occurrence, + sourceLine: null, + expected, + ...(reason ? { reason } : {}), + }, + exceptions + ) + + if (artifact.routeKind === "redirect") { + const target = (MARKDOWN_REDIRECT_TARGETS as Record)[requestPath] + if (!target) { + return { + findings: [ + finding( + "unverifiable", + "lang=default;synthetic=redirect;configuration", + requestPath, + "Redirect route has no independent checker target" + ), + ], + targetPaths: [], + } + } + const label = target + const url = `/${target}.md` + const observed = analyzeObservedMarkdown(artifact.markdown) + const present = observed.facts.some((fact) => fact.kind === "link" && fact.value === label && fact.url === url) + return { + findings: [ + finding( + present ? "present" : "missing", + `lang=default;synthetic=redirect;${label} -> ${url}`, + `${label} -> ${url}`, + present ? undefined : "Redirect artifact does not contain its exact current target link" + ), + ], + targetPaths: [target], + } + } + + if (artifact.routeKind === "selector") { + const targets = [`${requestPath}-go`, `${requestPath}-ts`] + const labels = ["Go", "TypeScript"] + const lines = artifact.markdown.split(/\r?\n/).map((line) => line.trim()) + return { + findings: targets.map((target, index) => { + const expectedLine = `- ${labels[index]}: /${target}.md` + const present = lines.includes(expectedLine) + return finding( + present ? "present" : "missing", + `lang=default;synthetic=selector;${labels[index]} -> /${target}.md`, + `${labels[index]} -> /${target}.md`, + present ? undefined : `Selector artifact is missing exact entry "${expectedLine}"` + ) + }), + targetPaths: targets, + } + } + + return { + findings: [ + finding( + "unverifiable", + `lang=default;synthetic=${artifact.routeKind};source`, + requestPath, + "Source-less artifact has no independent fidelity contract" + ), + ], + targetPaths: [], + } +} + +async function checkPathInternal( + requestPath: string, + ancestorPaths: ReadonlySet, + globallyScheduledPaths?: ReadonlySet, + globallyVisitedPaths?: Set +): Promise { + globallyVisitedPaths?.add(requestPath) + const nextAncestors = new Set(ancestorPaths) + nextAncestors.add(requestPath) + const defaultArtifact = await buildMarkdownArtifact(requestPath) + if (!defaultArtifact) { + return [ + withException( + { + path: requestPath, + status: "missing", + occurrence: "lang=default;artifact", + sourceLine: null, + reason: "No Markdown artifact was built", + }, + markdownFidelityExceptions + ), + ] + } + + const degraded = (artifact: MarkdownArtifact, lang: string): FidelityFinding[] => { + if (artifact.transformMode === "normal") return [] + const artifactSource = artifact.sourcePath ? safeSourcePath(artifact.sourcePath)?.relative : undefined + return [ + withException( + { + path: requestPath, + status: "degraded", + occurrence: `lang=${lang};transform=${artifact.transformMode}`, + sourcePath: artifactSource, + sourceLine: null, + ...(lang === "default" ? {} : { lang }), + reason: `${artifact.routeKind} route used ${artifact.transformMode} output`, + }, + markdownFidelityExceptions + ), + ] + } + + if (!defaultArtifact.sourcePath) { + const inspection = inspectSyntheticArtifact(requestPath, defaultArtifact) + const findings = [...degraded(defaultArtifact, "default"), ...inspection.findings] + for (const targetPath of inspection.targetPaths) { + if (nextAncestors.has(targetPath)) { + findings.push( + withException( + { + path: requestPath, + status: "unverifiable", + occurrence: `lang=default;synthetic-target-cycle=${targetPath}`, + sourceLine: null, + name: targetPath, + reason: "Synthetic route target evaluation forms a cycle", + }, + markdownFidelityExceptions + ) + ) + } else if (!globallyScheduledPaths?.has(targetPath) && !globallyVisitedPaths?.has(targetPath)) { + findings.push( + ...(await checkPathInternal(targetPath, nextAncestors, globallyScheduledPaths, globallyVisitedPaths)) + ) + } + } + return findings + } + const sourceLocation = safeSourcePath(defaultArtifact.sourcePath) + if (!sourceLocation) { + return [ + withException( + { + path: requestPath, + status: "unverifiable", + occurrence: "lang=default;source-path", + sourceLine: null, + name: defaultArtifact.sourcePath, + reason: "Artifact source path escapes src/content", + }, + markdownFidelityExceptions + ), + ] + } + + const source = await fs.readFile(sourceLocation.absolute, "utf8") + const analysis = analyzeSourceMarkdown(source, sourceLocation.relative) + const variants = ["default", ...analysis.languages] + const findings: FidelityFinding[] = [] + for (const lang of variants) { + const artifact = lang === "default" ? defaultArtifact : await buildMarkdownArtifact(requestPath, { lang }) + if (!artifact) { + findings.push( + withException( + { + path: requestPath, + status: "missing", + occurrence: `lang=${lang};artifact`, + sourcePath: sourceLocation.relative, + sourceLine: null, + ...(lang === "default" ? {} : { lang }), + reason: "No Markdown artifact was built for static language variant", + }, + markdownFidelityExceptions + ) + ) + continue + } + findings.push(...degraded(artifact, lang)) + findings.push( + ...compareSourceToArtifact( + requestPath, + sourceLocation.relative, + source, + artifact, + lang, + markdownFidelityExceptions + ) + ) + } + return findings +} + +export async function checkPath( + requestPath: string, + options: { globallyScheduledPaths?: ReadonlySet; globallyVisitedPaths?: Set } = {} +): Promise { + return checkPathInternal(requestPath, new Set(), options.globallyScheduledPaths, options.globallyVisitedPaths) +} + +export async function runMarkdownFidelity( + argv: readonly string[], + options: { reportPath?: string; contentRoot?: string } = {} +): Promise<{ report: FidelityReport; exitCode: 0 | 1 }> { + const parsed = parseCliArguments(argv) + const paths = parsed.mode === "focused" ? parsed.paths : await collectCorpusPaths(options.contentRoot) + const findings: FidelityFinding[] = [] + const globallyScheduledPaths = parsed.mode === "full-corpus" ? new Set(paths) : undefined + const globallyVisitedPaths = parsed.mode === "full-corpus" ? new Set() : undefined + for (const requestPath of paths) { + try { + findings.push(...(await checkPathInternal(requestPath, new Set(), globallyScheduledPaths, globallyVisitedPaths))) + } catch (error) { + const reason = (error instanceof Error ? error.message : "Checker failed").split(process.cwd()).join(".") + findings.push( + withException( + { + path: requestPath, + status: "unverifiable", + occurrence: "lang=default;checker-error", + sourceLine: null, + reason, + }, + markdownFidelityExceptions + ) + ) + } + } + const report = createReport(paths.length, findings) + const reportPath = options.reportPath ?? DEFAULT_REPORT_PATH + await fs.mkdir(path.dirname(reportPath), { recursive: true }) + await fs.writeFile(reportPath, serializeReport(report), "utf8") + return { report, exitCode: determineExitCode(parsed.mode, report.findings) } +} + +async function main(): Promise { + const { report, exitCode } = await runMarkdownFidelity(process.argv.slice(2)) + const counts = Object.entries(report.counts) + .map(([status, count]) => `${status}=${count}`) + .join(" ") + console.log(`Markdown fidelity: paths=${report.pathCount} ${counts}`) + process.exitCode = exitCode +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : error) + process.exitCode = 1 + }) +} diff --git a/src/scripts/markdown-fidelity-exceptions.ts b/src/scripts/markdown-fidelity-exceptions.ts new file mode 100644 index 00000000000..3dbb85eec99 --- /dev/null +++ b/src/scripts/markdown-fidelity-exceptions.ts @@ -0,0 +1,3 @@ +import type { FidelityException } from "./check-markdown-fidelity.js" + +export const markdownFidelityExceptions: FidelityException[] = []