From 7e52ae5907da711906dba40b9306520a3ff2f594 Mon Sep 17 00:00:00 2001 From: Web Artisan Date: Fri, 18 Sep 2026 18:50:44 +0300 Subject: [PATCH] feat(site): llms.txt and raw markdown docs --- apps/site/README.md | 2 +- apps/site/src/lib/llms.ts | 46 ++++++++++++++++++++++++ apps/site/src/lib/rehype-doc-links.ts | 6 ++-- apps/site/src/lib/site.ts | 2 ++ apps/site/src/pages/docs/[...slug].astro | 9 ++--- apps/site/src/pages/docs/[...slug].md.ts | 10 ++++++ apps/site/src/pages/docs/index.astro | 2 +- apps/site/src/pages/llms-full.txt.ts | 16 +++++++++ apps/site/src/pages/llms.txt.ts | 41 +++++++++++++++++++++ 9 files changed, 122 insertions(+), 12 deletions(-) create mode 100644 apps/site/src/lib/llms.ts create mode 100644 apps/site/src/pages/docs/[...slug].md.ts create mode 100644 apps/site/src/pages/llms-full.txt.ts create mode 100644 apps/site/src/pages/llms.txt.ts diff --git a/apps/site/README.md b/apps/site/README.md index f30180f..278bfb3 100644 --- a/apps/site/README.md +++ b/apps/site/README.md @@ -2,7 +2,7 @@ The landing page and the rendered documentation at junctio.org, built with Astro, Tailwind 4 and Vue islands. -The documentation pages are generated from `../../docs/*.md` at build time; nothing under `src/content` duplicates them. Use-case pages live in `src/content/use-cases` and are the only prose authored here. +The documentation pages are generated from `../../docs/*.md` at build time; nothing under `src/content` duplicates them. The same files feed `/llms.txt`, `/llms-full.txt` and the raw Markdown at `/docs/.md`. Use-case pages live in `src/content/use-cases` and are the only prose authored here. ```bash bun run dev:site # astro dev server on 4321 diff --git a/apps/site/src/lib/llms.ts b/apps/site/src/lib/llms.ts new file mode 100644 index 0000000..c210b9f --- /dev/null +++ b/apps/site/src/lib/llms.ts @@ -0,0 +1,46 @@ +import { resolve } from "node:path"; +import type { CollectionEntry } from "astro:content"; +import { DOCS_ROOT, rewriteDocLink } from "./rehype-doc-links"; +import { SITE } from "./site"; + +type Doc = CollectionEntry<"docs">; + +export function docSummary(entry: Doc): string { + return ( + entry.body + ?.split("\n") + .map((line) => line.trim()) + .find((line) => line.length > 0 && !line.startsWith("#") && !line.startsWith("|") && !line.startsWith("```")) + ?.replace(/[*`[\]]/g, "") ?? SITE.description + ); +} + +export function firstSentence(text: string): string { + return text.match(/^.*?[.!?](?=\s|$)/)?.[0] ?? text; +} + +export function docMarkdown(entry: Doc, site: URL): string { + const from = resolve(DOCS_ROOT, `${entry.id}.md`); + return (entry.body ?? "").replace(/\]\(([^)\s]+)\)/g, (_, href: string) => { + const next = rewriteDocLink(href, from); + return `](${next.startsWith("/") ? new URL(next, site).href : next})`; + }); +} + +export function demoteHeadings(markdown: string): string { + let fenced = false; + return markdown + .split("\n") + .map((line) => { + if (/^\s*(```|~~~)/.test(line)) fenced = !fenced; + return !fenced && /^#{1,5} /.test(line) ? `#${line}` : line; + }) + .join("\n"); +} + +export function llmsHeader(): string[] { + return [`# ${SITE.name}`, "", `> ${SITE.tagline}. ${SITE.description}`, ""]; +} + +export const TEXT = { headers: { "Content-Type": "text/plain; charset=utf-8" } }; +export const MARKDOWN = { headers: { "Content-Type": "text/markdown; charset=utf-8" } }; diff --git a/apps/site/src/lib/rehype-doc-links.ts b/apps/site/src/lib/rehype-doc-links.ts index cc930f9..530d6c5 100644 --- a/apps/site/src/lib/rehype-doc-links.ts +++ b/apps/site/src/lib/rehype-doc-links.ts @@ -5,10 +5,10 @@ import type { Element, Root } from "hast"; import type { VFile } from "vfile"; const REPO_ROOT = resolve(fileURLToPath(new URL("../../../..", import.meta.url))); -const DOCS_ROOT = resolve(REPO_ROOT, "docs"); +export const DOCS_ROOT = resolve(REPO_ROOT, "docs"); const GITHUB_BLOB = "https://github.com/k2so-dev/junctio/blob/main"; -function rewrite(href: string, fromFile: string): string { +export function rewriteDocLink(href: string, fromFile: string): string { if (/^[a-z]+:/i.test(href) || href.startsWith("#") || href.startsWith("/")) return href; const [pathPart, hash] = href.split("#", 2); if (!pathPart?.endsWith(".md")) return href; @@ -31,7 +31,7 @@ export function rehypeDocLinks() { if (node.tagName !== "a") return; const href = node.properties?.href; if (typeof href !== "string") return; - const next = rewrite(href, from); + const next = rewriteDocLink(href, from); node.properties.href = next; if (next.startsWith("http")) { node.properties.target = "_blank"; diff --git a/apps/site/src/lib/site.ts b/apps/site/src/lib/site.ts index 3553e72..14b993f 100644 --- a/apps/site/src/lib/site.ts +++ b/apps/site/src/lib/site.ts @@ -3,6 +3,8 @@ export const SITE = { tagline: "Self-hosted MCP gateway for one developer or a small team", description: "One endpoint for Claude Code, Cursor, Codex and claude.ai. Upstream OAuth tokens refreshed before they expire, no telemetry, one container.", + gettingStarted: + "Run the Junctio MCP gateway with Docker Compose, add an upstream server, and point Claude Code or Cursor at one endpoint.", repo: "https://github.com/k2so-dev/junctio", image: "ghcr.io/k2so-dev/junctio", composeUrl: "https://raw.githubusercontent.com/k2so-dev/junctio/main/compose.yml", diff --git a/apps/site/src/pages/docs/[...slug].astro b/apps/site/src/pages/docs/[...slug].astro index afc308c..aa44a04 100644 --- a/apps/site/src/pages/docs/[...slug].astro +++ b/apps/site/src/pages/docs/[...slug].astro @@ -1,5 +1,6 @@ --- import DocsLayout from "@/components/DocsLayout.astro"; +import { docSummary } from "@/lib/llms"; import { SITE } from "@/lib/site"; import { getCollection, render } from "astro:content"; @@ -11,13 +12,7 @@ export async function getStaticPaths() { const { entry } = Astro.props; const { Content, headings } = await render(entry); const title = headings.find((heading) => heading.depth === 1)?.text ?? entry.id; -const description = - entry.body - ?.split("\n") - .map((line) => line.trim()) - .find((line) => line.length > 0 && !line.startsWith("#") && !line.startsWith("|") && !line.startsWith("```")) - ?.replace(/[*`\[\]]/g, "") - .slice(0, 160) ?? SITE.description; +const description = docSummary(entry).slice(0, 160); --- ({ params: { slug: entry.id }, props: { entry } })); +} + +export const GET: APIRoute = ({ props, site }) => new Response(docMarkdown(props.entry, site!), MARKDOWN); diff --git a/apps/site/src/pages/docs/index.astro b/apps/site/src/pages/docs/index.astro index d514920..2e8041f 100644 --- a/apps/site/src/pages/docs/index.astro +++ b/apps/site/src/pages/docs/index.astro @@ -13,7 +13,7 @@ const headings = [ diff --git a/apps/site/src/pages/llms-full.txt.ts b/apps/site/src/pages/llms-full.txt.ts new file mode 100644 index 0000000..5abeff7 --- /dev/null +++ b/apps/site/src/pages/llms-full.txt.ts @@ -0,0 +1,16 @@ +import type { APIRoute } from "astro"; +import { getCollection } from "astro:content"; +import { TEXT, demoteHeadings, docMarkdown, llmsHeader } from "@/lib/llms"; +import { DOCS_NAV } from "@/lib/site"; + +export const GET: APIRoute = async ({ site }) => { + const docs = new Map((await getCollection("docs")).map((entry) => [entry.id, entry])); + const parts = [llmsHeader().join("\n").trimEnd()]; + + for (const item of DOCS_NAV.flatMap((group) => group.items)) { + const entry = docs.get(item.slug); + if (entry) parts.push(demoteHeadings(docMarkdown(entry, site!)).trim()); + } + + return new Response(`${parts.join("\n\n")}\n`, TEXT); +}; diff --git a/apps/site/src/pages/llms.txt.ts b/apps/site/src/pages/llms.txt.ts new file mode 100644 index 0000000..dbc15cb --- /dev/null +++ b/apps/site/src/pages/llms.txt.ts @@ -0,0 +1,41 @@ +import type { APIRoute } from "astro"; +import { getCollection } from "astro:content"; +import { TEXT, docSummary, firstSentence, llmsHeader } from "@/lib/llms"; +import { DOCS_NAV, SITE } from "@/lib/site"; + +export const GET: APIRoute = async ({ site }) => { + const docs = new Map((await getCollection("docs")).map((entry) => [entry.id, entry])); + const useCases = (await getCollection("useCases")).sort((a, b) => a.data.order - b.data.order); + const url = (path: string) => new URL(path, site).href; + const lines = llmsHeader(); + + for (const group of DOCS_NAV) { + lines.push(`## ${group.label}`, ""); + for (const item of group.items) { + const entry = docs.get(item.slug); + if (!entry) { + lines.push(`- [${item.title}](${url("/docs/")}): ${SITE.gettingStarted}`); + continue; + } + lines.push(`- [${item.title}](${url(`/docs/${item.slug}.md`)}): ${firstSentence(docSummary(entry))}`); + } + lines.push(""); + } + + lines.push("## Use cases", ""); + for (const entry of useCases) { + lines.push(`- [${entry.data.title}](${url(`/use-cases/${entry.id}/`)}): ${entry.data.description}`); + } + + lines.push( + "", + "## Optional", + "", + `- [Full documentation](${url("/llms-full.txt")}): every docs page in one file`, + `- [Source](${SITE.repo})`, + `- [Container image](${SITE.repo}/pkgs/container/junctio): ${SITE.image}, latest is the last release, edge follows main`, + "" + ); + + return new Response(lines.join("\n"), TEXT); +};