Preserve Markdown route fidelity

package.json

@@ -52,7 +52,8 @@
     "quality-check": "npm run typecheck && npm run lint && npm run audit:exports",
     "quality-check:full": "npm run typecheck && npm run lint && npm run audit:dead-code",
     "generate:llms": "tsx --require tsconfig-paths/register src/scripts/generate-llms.ts",
-    "check:llms": "tsx --require tsconfig-paths/register src/scripts/validate-llms.ts"
+    "check:llms": "tsx --require tsconfig-paths/register src/scripts/validate-llms.ts",
+    "check:markdown-fidelity": "tsx --require tsconfig-paths/register src/scripts/check-markdown-fidelity.ts"
   },
   "dependencies": {
     "@11ty/eleventy-fetch": "^4.0.1",

src/lib/markdown/__tests__/buildMarkdownArtifact.test.ts

@@ -0,0 +1,125 @@
+import { describe, expect, it } from "@jest/globals"
+import {
+  buildMarkdownArtifact,
+  normalizeMarkdownPath,
+  transformPageBodyToMarkdown,
+} from "@lib/markdown/buildMarkdownArtifact.js"
+
+describe("buildMarkdownArtifact", () => {
+  it.each([
+    ["cre/getting-started/cli-installation", "normal"],
+    ["cre-templates", "special"],
+    ["cre/reference/sdk/evm-client", "selector"],
+    ["data-streams/getting-started", "redirect"],
+  ])("classifies %s as %s", async (requestPath, routeKind) => {
+    const artifact = await buildMarkdownArtifact(requestPath)
+
+    expect(artifact).not.toBeNull()
+    expect(artifact?.requestPath).toBe(normalizeMarkdownPath(requestPath))
+    expect(artifact?.routeKind).toBe(routeKind)
+  })
+
+  it("rejects path escapes", async () => {
+    expect(normalizeMarkdownPath("../outside")).toBeNull()
+    await expect(buildMarkdownArtifact("../outside")).resolves.toBeNull()
+  })
+
+  it("trims long leading and trailing slash runs", () => {
+    const slashes = "/".repeat(100_000)
+
+    expect(normalizeMarkdownPath(`${slashes}cre/getting-started${slashes}`)).toBe("cre/getting-started")
+  })
+
+  it("accepts an existing extensionless production request path", async () => {
+    await expect(buildMarkdownArtifact("cre/getting-started/cli-installation")).resolves.toMatchObject({
+      requestPath: "cre/getting-started/cli-installation",
+      routeKind: "normal",
+    })
+  })
+
+  it.each([".md", ".md.md", ".mdx"])("rejects a leftover %s extension", async (extension) => {
+    await expect(buildMarkdownArtifact(`cre/getting-started/cli-installation${extension}`)).resolves.toBeNull()
+  })
+
+  it.each([
+    "cre/getting-started/cli-installation",
+    "cre/getting-started/cli-installation/macos-linux",
+    "cre/getting-started/cli-installation/windows",
+  ])("projects the ordered operating system selector for %s", async (requestPath) => {
+    const artifact = await buildMarkdownArtifact(requestPath)
+    const markdown = artifact?.markdown ?? ""
+    const macosLinux = "[macOS / Linux](/cre/getting-started/cli-installation/macos-linux)"
+    const windows = "[Windows](/cre/getting-started/cli-installation/windows)"
+
+    expect(markdown).toContain("## Select your operating system")
+    expect(markdown).toContain(macosLinux)
+    expect(markdown).toContain(windows)
+    expect(markdown.indexOf(macosLinux)).toBeLessThan(markdown.indexOf(windows))
+  })
+})
+
+describe("transformPageBodyToMarkdown", () => {
+  it("reports the normal transform branch", async () => {
+    const result = await transformPageBodyToMarkdown("# Kept", "/virtual/normal.mdx")
+
+    expect(result.transformMode).toBe("normal")
+    expect(result.markdown).toContain("# Kept")
+  })
+
+  it("reports the sanitized retry branch", async () => {
+    const body = `export async function load() {
+  return @
+}
+
+# Kept`
+    const result = await transformPageBodyToMarkdown(body, "/virtual/sanitized.mdx")
+
+    expect(result.transformMode).toBe("sanitized")
+    expect(result.markdown).toContain("# Kept")
+    expect(result.markdown).not.toContain("return @")
+  })
+
+  it("reports the fallback branch", async () => {
+    const body = `# Kept
+
+{`
+    const result = await transformPageBodyToMarkdown(body, "/virtual/fallback.mdx")
+
+    expect(result).toEqual({
+      transformMode: "fallback",
+      markdown: body,
+    })
+  })
+
+  it("strips component tags in the fallback branch", async () => {
+    const result = await transformPageBodyToMarkdown(
+      `<Wrapper data-label="a = b"><Callout />Visible</Wrapper>
+{`,
+      "/virtual/fallback-components.mdx"
+    )
+
+    expect(result).toEqual({
+      transformMode: "fallback",
+      markdown: `Visible
+{`,
+    })
+  })
+
+  it("preserves a long unterminated repeated component prefix in the fallback branch", async () => {
+    const body = `${"<A".repeat(10_000)}
+{`
+    const result = await transformPageBodyToMarkdown(body, "/virtual/fallback-malformed-components.mdx")
+
+    expect(result).toEqual({
+      transformMode: "fallback",
+      markdown: body,
+    })
+  })
+
+  it("reports the deprecating feeds replacement branch", async () => {
+    const result = await transformPageBodyToMarkdown("ignored", "/virtual/data-feeds/deprecating-feeds.mdx")
+
+    expect(result.transformMode).toBe("replacement")
+    expect(result.markdown).toContain("## Deprecated Feeds")
+  })
+})

src/lib/markdown/__tests__/sourceScanners.test.ts

@@ -0,0 +1,86 @@
+import { describe, expect, it } from "@jest/globals"
+import {
+  readStaticDefaultImports,
+  readStaticJsxSelectorConditions,
+  removeLeadingMdxFrontmatter,
+  stripHighlighterComments,
+} from "@lib/markdown/sourceScanners.js"
+
+describe("readStaticDefaultImports", () => {
+  it("reads single-line and multiline static default imports", () => {
+    const imports = readStaticDefaultImports(`---
+import Alpha from "./alpha.mdx"
+import $Code
+  from
+  './code.ts?raw'
+import { ignored } from "./named.js"
+import "./side-effect.js"
+---`)
+
+    expect(Object.fromEntries(imports)).toEqual({
+      Alpha: "./alpha.mdx",
+      $Code: "./code.ts?raw",
+    })
+  })
+
+  it("scans repeated unterminated import prefixes deterministically", () => {
+    const source = `${'import Broken from "unterminated\n'.repeat(10_000)}import Kept from "./kept.mdx"`
+
+    expect(Object.fromEntries(readStaticDefaultImports(source))).toEqual({ Kept: "./kept.mdx" })
+    expect(Object.fromEntries(readStaticDefaultImports(source))).toEqual({ Kept: "./kept.mdx" })
+  })
+})
+
+describe("readStaticJsxSelectorConditions", () => {
+  it("maps static selector values to JSX components", () => {
+    const conditions = readStaticJsxSelectorConditions(
+      `{callout === "alpha" && <Alpha />}
+{callout
+  ===
+  'beta'
+  &&
+  <Beta />}`,
+      "callout"
+    )
+
+    expect(Object.fromEntries(conditions)).toEqual({ alpha: "Alpha", beta: "Beta" })
+  })
+
+  it("scans repeated unterminated selector prefixes deterministically", () => {
+    const source = `${'{callout === "unterminated\n'.repeat(10_000)}{callout === "kept" && <Kept />}`
+
+    expect(Object.fromEntries(readStaticJsxSelectorConditions(source, "callout"))).toEqual({ kept: "Kept" })
+    expect(Object.fromEntries(readStaticJsxSelectorConditions(source, "callout"))).toEqual({ kept: "Kept" })
+  })
+})
+
+describe("removeLeadingMdxFrontmatter", () => {
+  it.each([
+    ["LF", "---\ntitle: Example\n---\n\n# Body\n", "\n# Body\n"],
+    ["CRLF", "---\r\ntitle: Example\r\n---\r\n# Body\r\n", "# Body\r\n"],
+    ["trailing fence whitespace", "---  \ntitle: Example\n--- \n# Body", "# Body"],
+  ])("removes leading %s frontmatter without changing body newlines", (_name, source, expected) => {
+    expect(removeLeadingMdxFrontmatter(source)).toBe(expected)
+  })
+
+  it("preserves missing and unterminated frontmatter", () => {
+    expect(removeLeadingMdxFrontmatter("# Body\n---\n")).toBe("# Body\n---\n")
+    expect(removeLeadingMdxFrontmatter("---\ntitle: Example")).toBe("---\ntitle: Example")
+  })
+})
+
+describe("stripHighlighterComments", () => {
+  it("removes supported markers while preserving other text and whitespace-only lines", () => {
+    const code = `const value = 1 // highlight-line
+
+\t// highlight-start
+next // regular comment
+end   //   highlight-end  `
+
+    expect(stripHighlighterComments(code)).toBe(`const value = 1
+
+
+next // regular comment
+end  `)
+  })
+})