feat(site): llms.txt and raw markdown docs

apps/site/README.md

@@ -2,7 +2,7 @@
 
 The landing page and the rendered documentation at junctio.org, built with Astro, Tailwind 4 and Vue islands.
 
-The documentation pages are generated from `../../docs/*.md` at build time; nothing under `src/content` duplicates them. Use-case pages live in `src/content/use-cases` and are the only prose authored here.
+The documentation pages are generated from `../../docs/*.md` at build time; nothing under `src/content` duplicates them. The same files feed `/llms.txt`, `/llms-full.txt` and the raw Markdown at `/docs/<slug>.md`. Use-case pages live in `src/content/use-cases` and are the only prose authored here.
 
 ```bash
 bun run dev:site     # astro dev server on 4321

apps/site/src/lib/llms.ts

@@ -0,0 +1,46 @@
+import { resolve } from "node:path";
+import type { CollectionEntry } from "astro:content";
+import { DOCS_ROOT, rewriteDocLink } from "./rehype-doc-links";
+import { SITE } from "./site";
+
+type Doc = CollectionEntry<"docs">;
+
+export function docSummary(entry: Doc): string {
+  return (
+    entry.body
+      ?.split("\n")
+      .map((line) => line.trim())
+      .find((line) => line.length > 0 && !line.startsWith("#") && !line.startsWith("|") && !line.startsWith("```"))
+      ?.replace(/[*`[\]]/g, "") ?? SITE.description
+  );
+}
+
+export function firstSentence(text: string): string {
+  return text.match(/^.*?[.!?](?=\s|$)/)?.[0] ?? text;
+}
+
+export function docMarkdown(entry: Doc, site: URL): string {
+  const from = resolve(DOCS_ROOT, `${entry.id}.md`);
+  return (entry.body ?? "").replace(/\]\(([^)\s]+)\)/g, (_, href: string) => {
+    const next = rewriteDocLink(href, from);
+    return `](${next.startsWith("/") ? new URL(next, site).href : next})`;
+  });
+}
+
+export function demoteHeadings(markdown: string): string {
+  let fenced = false;
+  return markdown
+    .split("\n")
+    .map((line) => {
+      if (/^\s*(```|~~~)/.test(line)) fenced = !fenced;
+      return !fenced && /^#{1,5} /.test(line) ? `#${line}` : line;
+    })
+    .join("\n");
+}
+
+export function llmsHeader(): string[] {
+  return [`# ${SITE.name}`, "", `> ${SITE.tagline}. ${SITE.description}`, ""];
+}
+
+export const TEXT = { headers: { "Content-Type": "text/plain; charset=utf-8" } };
+export const MARKDOWN = { headers: { "Content-Type": "text/markdown; charset=utf-8" } };

apps/site/src/lib/rehype-doc-links.ts

@@ -5,10 +5,10 @@ import type { Element, Root } from "hast";
 import type { VFile } from "vfile";
 
 const REPO_ROOT = resolve(fileURLToPath(new URL("../../../..", import.meta.url)));
-const DOCS_ROOT = resolve(REPO_ROOT, "docs");
+export const DOCS_ROOT = resolve(REPO_ROOT, "docs");
 const GITHUB_BLOB = "https://github.com/k2so-dev/junctio/blob/main";
 
-function rewrite(href: string, fromFile: string): string {
+export function rewriteDocLink(href: string, fromFile: string): string {
   if (/^[a-z]+:/i.test(href) || href.startsWith("#") || href.startsWith("/")) return href;
   const [pathPart, hash] = href.split("#", 2);
   if (!pathPart?.endsWith(".md")) return href;
@@ -31,7 +31,7 @@ export function rehypeDocLinks() {
       if (node.tagName !== "a") return;
       const href = node.properties?.href;
       if (typeof href !== "string") return;
-      const next = rewrite(href, from);
+      const next = rewriteDocLink(href, from);
       node.properties.href = next;
       if (next.startsWith("http")) {
         node.properties.target = "_blank";

apps/site/src/lib/site.ts

@@ -3,6 +3,8 @@ export const SITE = {
   tagline: "Self-hosted MCP gateway for one developer or a small team",
   description:
     "One endpoint for Claude Code, Cursor, Codex and claude.ai. Upstream OAuth tokens refreshed before they expire, no telemetry, one container.",
+  gettingStarted:
+    "Run the Junctio MCP gateway with Docker Compose, add an upstream server, and point Claude Code or Cursor at one endpoint.",
   repo: "https://github.com/k2so-dev/junctio",
   image: "ghcr.io/k2so-dev/junctio",
   composeUrl: "https://raw.githubusercontent.com/k2so-dev/junctio/main/compose.yml",

apps/site/src/pages/docs/[...slug].astro

@@ -1,5 +1,6 @@
 ---
 import DocsLayout from "@/components/DocsLayout.astro";
+import { docSummary } from "@/lib/llms";
 import { SITE } from "@/lib/site";
 import { getCollection, render } from "astro:content";
 
@@ -11,13 +12,7 @@ export async function getStaticPaths() {
 const { entry } = Astro.props;
 const { Content, headings } = await render(entry);
 const title = headings.find((heading) => heading.depth === 1)?.text ?? entry.id;
-const description =
-  entry.body
-    ?.split("\n")
-    .map((line) => line.trim())
-    .find((line) => line.length > 0 && !line.startsWith("#") && !line.startsWith("|") && !line.startsWith("```"))
-    ?.replace(/[*`\[\]]/g, "")
-    .slice(0, 160) ?? SITE.description;
+const description = docSummary(entry).slice(0, 160);
 ---
 
 <DocsLayout

apps/site/src/pages/docs/[...slug].md.ts

@@ -0,0 +1,10 @@
+import type { APIRoute } from "astro";
+import { getCollection } from "astro:content";
+import { MARKDOWN, docMarkdown } from "@/lib/llms";
+
+export async function getStaticPaths() {
+  const entries = await getCollection("docs");
+  return entries.map((entry) => ({ params: { slug: entry.id }, props: { entry } }));
+}
+
+export const GET: APIRoute = ({ props, site }) => new Response(docMarkdown(props.entry, site!), MARKDOWN);

apps/site/src/pages/docs/index.astro

@@ -13,7 +13,7 @@ const headings = [
 
 <DocsLayout
   title="Getting started"
-  description="Run the Junctio MCP gateway with Docker Compose, add an upstream server, and point Claude Code or Cursor at one endpoint."
+  description={SITE.gettingStarted}
   current=""
   headings={headings}
 >

apps/site/src/pages/llms-full.txt.ts

@@ -0,0 +1,16 @@
+import type { APIRoute } from "astro";
+import { getCollection } from "astro:content";
+import { TEXT, demoteHeadings, docMarkdown, llmsHeader } from "@/lib/llms";
+import { DOCS_NAV } from "@/lib/site";
+
+export const GET: APIRoute = async ({ site }) => {
+  const docs = new Map((await getCollection("docs")).map((entry) => [entry.id, entry]));
+  const parts = [llmsHeader().join("\n").trimEnd()];
+
+  for (const item of DOCS_NAV.flatMap((group) => group.items)) {
+    const entry = docs.get(item.slug);
+    if (entry) parts.push(demoteHeadings(docMarkdown(entry, site!)).trim());
+  }
+
+  return new Response(`${parts.join("\n\n")}\n`, TEXT);
+};

apps/site/src/pages/llms.txt.ts

@@ -0,0 +1,41 @@
+import type { APIRoute } from "astro";
+import { getCollection } from "astro:content";
+import { TEXT, docSummary, firstSentence, llmsHeader } from "@/lib/llms";
+import { DOCS_NAV, SITE } from "@/lib/site";
+
+export const GET: APIRoute = async ({ site }) => {
+  const docs = new Map((await getCollection("docs")).map((entry) => [entry.id, entry]));
+  const useCases = (await getCollection("useCases")).sort((a, b) => a.data.order - b.data.order);
+  const url = (path: string) => new URL(path, site).href;
+  const lines = llmsHeader();
+
+  for (const group of DOCS_NAV) {
+    lines.push(`## ${group.label}`, "");
+    for (const item of group.items) {
+      const entry = docs.get(item.slug);
+      if (!entry) {
+        lines.push(`- [${item.title}](${url("/docs/")}): ${SITE.gettingStarted}`);
+        continue;
+      }
+      lines.push(`- [${item.title}](${url(`/docs/${item.slug}.md`)}): ${firstSentence(docSummary(entry))}`);
+    }
+    lines.push("");
+  }
+
+  lines.push("## Use cases", "");
+  for (const entry of useCases) {
+    lines.push(`- [${entry.data.title}](${url(`/use-cases/${entry.id}/`)}): ${entry.data.description}`);
+  }
+
+  lines.push(
+    "",
+    "## Optional",
+    "",
+    `- [Full documentation](${url("/llms-full.txt")}): every docs page in one file`,
+    `- [Source](${SITE.repo})`,
+    `- [Container image](${SITE.repo}/pkgs/container/junctio): ${SITE.image}, latest is the last release, edge follows main`,
+    ""
+  );
+
+  return new Response(lines.join("\n"), TEXT);
+};