diff --git a/bun.lock b/bun.lock index dfc62086add2..591ed697e5ad 100644 --- a/bun.lock +++ b/bun.lock @@ -701,6 +701,7 @@ "@opencode/ui": "workspace:*", "@opencode/util": "workspace:*", "@pierre/diffs": "catalog:", + "@pierre/highlights": "catalog:", "@shikijs/stream": "catalog:", "@solid-primitives/event-listener": "catalog:", "@solid-primitives/media": "catalog:", @@ -1097,7 +1098,8 @@ "@opentui/core": "0.5.17", "@opentui/keymap": "0.5.17", "@opentui/solid": "0.5.17", - "@pierre/diffs": "1.5.1", + "@pierre/diffs": "1.6.1", + "@pierre/highlights": "0.2.0", "@playwright/test": "1.59.1", "@sentry/solid": "10.71.0", "@sentry/vite-plugin": "5.4.0", @@ -2408,7 +2410,9 @@ "@peculiar/webcrypto": ["@peculiar/webcrypto@1.7.1", "", { "dependencies": { "@peculiar/asn1-schema": "^2.7.0", "@peculiar/json-schema": "^1.1.12", "@peculiar/utils": "^2.0.2", "tslib": "^2.8.1", "webcrypto-core": "^1.9.2" } }, "sha512-ODOov0sGMJMf3jPonOkgGqPknTsu+DdQ7kD++gz8aI+aFMOMHFbWAA2taqXXVTdP+OTOQR/znGvSpmkeI0WTYQ=="], - "@pierre/diffs": ["@pierre/diffs@1.5.1", "", { "dependencies": { "@pierre/theme": "2.0.0", "@pierre/theming": "1.0.1", "@shikijs/transformers": "^3.0.0 || ^4.0.0", "diff": "9.0.0", "hast-util-to-html": "9.0.5", "lru_map": "0.4.1", "shiki": "^3.0.0 || ^4.0.0" }, "peerDependencies": { "react": "^18.3.1 || ^19.0.0", "react-dom": "^18.3.1 || ^19.0.0" }, "optionalPeers": ["react", "react-dom"] }, "sha512-+EXNfz4ZXI6FHr7P6ToWEORwyYiPVfAmzhA70xF+0y/oNvj7gbNTLJJ6jNzkBjuMbRlu564TAmgYZLc5OyHURA=="], + "@pierre/diffs": ["@pierre/diffs@1.6.1", "", { "dependencies": { "@pierre/highlights": "0.2.0", "@pierre/theme": "2.0.0", "@pierre/theming": "1.0.2", "diff": "9.0.0", "hast-util-to-html": "9.0.5", "lru_map": "0.4.1", "shiki": "^3.0.0 || ^4.0.0" }, "peerDependencies": { "react": "^18.3.1 || ^19.0.0", "react-dom": "^18.3.1 || ^19.0.0" }, "optionalPeers": ["react", "react-dom"] }, "sha512-XTfLW4bFETyosZfFNUoyRh1zPE4tmAijOoxdHx4X805tlaq/7/fuQXIiJSm21+jLkDo8vn5elQ+2O4YtMTKM0Q=="], + + "@pierre/highlights": ["@pierre/highlights@0.2.0", "", {}, "sha512-hTpcOU1EzDAP4bYtEhxmMu2jwMvtkUydRUBUpWMSHTs3dw0IaC3Ld5ansMy18L0GE3b6Vfl2OGkLK+pWWpnBmw=="], "@pierre/theme": ["@pierre/theme@2.0.0", "", {}, "sha512-yNDd9GYLQl1mEUJR8AneJ5e4ohLIHQd/wZLWr4fagt78vS2RwwZNW530vVgHqXFAyFVcFlRmGUD5ramXH46OXw=="], @@ -5672,7 +5676,7 @@ "@paulirish/trace_engine/third-party-web": ["third-party-web@0.30.0", "", {}, "sha512-p+PfyL5U0ediGzvwPzMSdD9MnC8rHW6nWJJVLIQvqN7RsSqfaZ+7n4c9GILqH36EYR0eXB9CzAeCRnKFr/9uJg=="], - "@pierre/diffs/@pierre/theming": ["@pierre/theming@1.0.1", "", { "peerDependencies": { "@pierre/theme": "^1.1.0 || ^2.0.0", "@shikijs/themes": "^3.0.0 || ^4.0.0", "react": "^18.3.1 || ^19.0.0", "react-dom": "^18.3.1 || ^19.0.0", "shiki": "^3.0.0 || ^4.0.0" }, "optionalPeers": ["@pierre/theme", "@shikijs/themes", "react", "react-dom", "shiki"] }, "sha512-WCI5Qd7iprDpISL9fBYOLe8RV53+b7mFNA3bPzl60/2CKCSrsKN8zEcep6Y3BAzvARlmca50zGjDodqPGiTUKA=="], + "@pierre/diffs/@pierre/theming": ["@pierre/theming@1.0.2", "", { "peerDependencies": { "@pierre/theme": "^1.1.0 || ^2.0.0", "@shikijs/themes": "^3.0.0 || ^4.0.0", "react": "^18.3.1 || ^19.0.0", "react-dom": "^18.3.1 || ^19.0.0", "shiki": "^3.0.0 || ^4.0.0" }, "optionalPeers": ["@pierre/theme", "@shikijs/themes", "react", "react-dom", "shiki"] }, "sha512-RmXLLqtx31UGcR8kZyO5hGAQ1EXz6loT1Cu0BFAGsbnBboIxp2BlomkLfWyvdT4WZXnBGyu25onL1gKpFQuSUw=="], "@pierre/diffs/diff": ["diff@9.0.0", "", {}, "sha512-svtcdpS8CgJyqAjEQIXdb3OjhFVVYjzGAPO8WGCmRbrml64SPw/jJD4GoE98aR7r25A0XcgrK3F02yw9R/vhQw=="], diff --git a/bunfig.toml b/bunfig.toml index c75f6536ee1b..2c9e551361e5 100644 --- a/bunfig.toml +++ b/bunfig.toml @@ -2,7 +2,7 @@ exact = true # Only install newly resolved package versions published at least 3 days ago. minimumReleaseAge = 259200 -minimumReleaseAgeExcludes = ["@ai-sdk/amazon-bedrock", "@ai-sdk/anthropic", "@brendonovich/vite-plugin-opencode", "@kitlangton/solid-motion", "@opencode/sdk", "@opencode-ai/pty", "@opencode-ai/pty-darwin-arm64", "@opencode-ai/pty-darwin-x64", "@opencode-ai/pty-linux-arm64-gnu", "@opencode-ai/pty-linux-arm64-musl", "@opencode-ai/pty-linux-x64-gnu", "@opencode-ai/pty-linux-x64-musl", "@opentui/core", "@opentui/core-darwin-arm64", "@opentui/core-darwin-x64", "@opentui/core-linux-arm64", "@opentui/core-linux-arm64-musl", "@opentui/core-linux-x64", "@opentui/core-linux-x64-musl", "@opentui/core-win32-arm64", "@opentui/core-win32-x64", "@opentui/keymap", "@opentui/solid", "@opentunnel/client", "@opentunnel/protocol", "opentui-spinner", "gitlab-ai-provider", "opencode-gitlab-auth", "@ff-labs/fff-node", "@ff-labs/fff-bun", "@ff-labs/fff-bin-darwin-arm64", "@ff-labs/fff-bin-darwin-x64", "@ff-labs/fff-bin-linux-arm64-gnu", "@ff-labs/fff-bin-linux-arm64-musl", "@ff-labs/fff-bin-linux-x64-gnu", "@ff-labs/fff-bin-linux-x64-musl", "@ff-labs/fff-bin-win32-arm64", "@ff-labs/fff-bin-win32-x64", "@pierre/diffs", "@pierre/theming", "app-builder-lib", "dmg-builder", "electron", "electron-builder", "electron-publish", "blume", "pele"] +minimumReleaseAgeExcludes = ["@ai-sdk/amazon-bedrock", "@ai-sdk/anthropic", "@brendonovich/vite-plugin-opencode", "@kitlangton/solid-motion", "@opencode/sdk", "@opencode-ai/pty", "@opencode-ai/pty-darwin-arm64", "@opencode-ai/pty-darwin-x64", "@opencode-ai/pty-linux-arm64-gnu", "@opencode-ai/pty-linux-arm64-musl", "@opencode-ai/pty-linux-x64-gnu", "@opencode-ai/pty-linux-x64-musl", "@opentui/core", "@opentui/core-darwin-arm64", "@opentui/core-darwin-x64", "@opentui/core-linux-arm64", "@opentui/core-linux-arm64-musl", "@opentui/core-linux-x64", "@opentui/core-linux-x64-musl", "@opentui/core-win32-arm64", "@opentui/core-win32-x64", "@opentui/keymap", "@opentui/solid", "@opentunnel/client", "@opentunnel/protocol", "opentui-spinner", "gitlab-ai-provider", "opencode-gitlab-auth", "@ff-labs/fff-node", "@ff-labs/fff-bun", "@ff-labs/fff-bin-darwin-arm64", "@ff-labs/fff-bin-darwin-x64", "@ff-labs/fff-bin-linux-arm64-gnu", "@ff-labs/fff-bin-linux-arm64-musl", "@ff-labs/fff-bin-linux-x64-gnu", "@ff-labs/fff-bin-linux-x64-musl", "@ff-labs/fff-bin-win32-arm64", "@ff-labs/fff-bin-win32-x64", "@pierre/diffs", "@pierre/highlights", "@pierre/theming", "app-builder-lib", "dmg-builder", "electron", "electron-builder", "electron-publish", "blume", "pele"] [test] root = "./do-not-run-tests-from-root" diff --git a/package.json b/package.json index e8d43fd18be8..4adff2ffbadb 100644 --- a/package.json +++ b/package.json @@ -69,7 +69,8 @@ "@tsconfig/node22": "22.0.2", "@tsconfig/bun": "1.0.9", "@cloudflare/workers-types": "4.20251008.0", - "@pierre/diffs": "1.5.1", + "@pierre/diffs": "1.6.1", + "@pierre/highlights": "0.2.0", "opentui-spinner": "0.0.7", "@solid-primitives/event-listener": "2.4.6", "@solid-primitives/media": "2.3.6", diff --git a/packages/app/e2e/performance/highlighter/bundle.ts b/packages/app/e2e/performance/highlighter/bundle.ts new file mode 100644 index 000000000000..4708e7c914b0 --- /dev/null +++ b/packages/app/e2e/performance/highlighter/bundle.ts @@ -0,0 +1,61 @@ +import { readdir, readFile } from "node:fs/promises" +import path from "node:path" +import { gzipSync } from "node:zlib" + +// Sizes of the built app's highlighting assets: `bun bundle.ts `. Shiki ships its oniguruma engine and one +// chunk per grammar or theme, loaded on demand; Highlights ships one chunk with its WebAssembly inlined. +const dist = process.argv[2] ?? "dist" + +const assets = path.join(dist, "_assets") + +const files = await readdir(assets, { recursive: true }) + +const all = await Promise.all( + files + .filter((file) => /\.(js|mjs|wasm)$/.test(file)) + .map(async (file) => { + const bytes = await readFile(path.join(assets, file)) + + return { + file, + bytes: bytes.length, + gzip: gzipSync(bytes).length, + // The start of Highlights' inlined WebAssembly module, as base64. + highlights: bytes.includes("AGFzbQEAAAABWw5gAX8Bf2AAAGACf38Bf2ABfw"), + } + }), +) + +const total = (items: typeof all) => ({ + files: items.length, + kb: Math.round(items.reduce((sum, item) => sum + item.bytes, 0) / 1024), + gzipKB: Math.round(items.reduce((sum, item) => sum + item.gzip, 0) / 1024), +}) + +const highlights = all.filter((item) => item.highlights) + +const onig = all.filter((item) => /^wasm-|onig/i.test(path.basename(item.file)) && !item.highlights) + +const workers = all.filter((item) => /worker/i.test(item.file)) + +console.log( + JSON.stringify( + { + all: total(all), + highlights: { + ...total(highlights), + files: highlights.map((item) => `${item.file} ${Math.round(item.bytes / 1024)} KB`), + }, + oniguruma: { ...total(onig), files: onig.map((item) => `${item.file} ${Math.round(item.bytes / 1024)} KB`) }, + workers: workers.map( + (item) => `${item.file} ${Math.round(item.bytes / 1024)} KB (${Math.round(item.gzip / 1024)} KB gz)`, + ), + largest: all + .toSorted((a, b) => b.bytes - a.bytes) + .slice(0, 25) + .map((item) => `${item.file} ${Math.round(item.bytes / 1024)} KB (${Math.round(item.gzip / 1024)} KB gz)`), + }, + null, + 2, + ), +) diff --git a/packages/app/e2e/performance/highlighter/highlighter-benchmark.spec.ts b/packages/app/e2e/performance/highlighter/highlighter-benchmark.spec.ts new file mode 100644 index 000000000000..664b0785ee4b --- /dev/null +++ b/packages/app/e2e/performance/highlighter/highlighter-benchmark.spec.ts @@ -0,0 +1,403 @@ +import type { Browser, Page } from "@playwright/test" +import type { OpenCodeEvent } from "@opencode/client/promise" +import { mkdir, writeFile } from "node:fs/promises" +import path from "node:path" +import { openReviewLauncher } from "../../utils/side-panel" +import { benchmark, expect } from "../benchmark" +import { benchmarkLocation, setupTimelineBenchmark, textPartID } from "../timeline/session-timeline-benchmark.fixture" +import { largeSource, sampleReply, samples } from "./samples" + +// Shiki against Pierre Highlights, switched through Knobs → Experiments ("Code highlighter"): a long code-heavy +// reply streaming into the timeline, a ~490 KB TypeScript diff opening in the review pane, and (with +// HIGHLIGHTER_STILLS set to a directory) stills plus computed token colours of the same code in light and dark. + +const engines = ["shiki", "highlights"] as const + +type Engine = (typeof engines)[number] + +const sessionID = "ses_timeline_state_regression" + +const assistantMessageID = textPartID.split(":")[0]! + +const throttles = (process.env.HIGHLIGHTER_THROTTLES ?? "1,4").split(",").map(Number) + +let sequence = 0 + +for (const engine of engines) { + for (const throttle of throttles) { + benchmark(`streams a code-heavy reply with ${engine} at ${throttle}x CPU`, async ({ page, browser, report }) => { + benchmark.setTimeout(300_000) + const loaded = await useEngine(page, engine) + const marker = `stream-complete-${engine}` + const reply = sampleReply(Number(process.env.HIGHLIGHTER_REPEAT ?? 3), marker) + const deltas = chunk(reply, Number(process.env.HIGHLIGHTER_DELTA_CHARS ?? 48)) + const fixture = await setupTimelineBenchmark(page, { historyTurns: 2, eventBatch: 1 }) + fixture.transport.enqueue(textEvents(["Streaming"])) + await expect(fixture.text).toContainText("Streaming") + await fixture.waitForStableGeometry() + + const cdp = await page.context().newCDPSession(page) + await cdp.send("Emulation.setCPUThrottlingRate", { rate: throttle }) + await browser.startTracing(page, { categories: traceCategories }) + const started = Date.now() + fixture.transport.enqueue(deltas.map((delta) => textEvent(delta))) + await expect(fixture.text).toContainText(marker, { timeout: 240_000 }) + const streamMs = Date.now() - started + await fixture.waitForStableGeometry() + const trace = summarizeTrace(await browser.stopTracing()) + await cdp.send("Emulation.setCPUThrottlingRate", { rate: 1 }) + const memory = await measureMemory(page, browser) + + const dom = await fixture.text.evaluate((root) => ({ + codeBlocks: root.querySelectorAll('[data-component="markdown-code"]').length, + tokenSpans: root.querySelectorAll('[data-component="markdown-code"] code span').length, + })) + + expect(dom.codeBlocks).toBe(samples.length * Number(process.env.HIGHLIGHTER_REPEAT ?? 3)) + report( + { streamMs, ...trace, ...memory, ...dom, highlightsLoaded: loaded.highlights() }, + { engine, throttle, deltas: deltas.length, replyChars: reply.length }, + ) + }) + } + + benchmark(`opens a 490 KB TypeScript diff with ${engine}`, async ({ page, browser, report }) => { + benchmark.setTimeout(480_000) + const loaded = await useEngine(page, engine) + const before = largeSource(Number(process.env.HIGHLIGHTER_DIFF_BYTES ?? 490_000)) + const after = before.replace(/limit = 200/g, (match, offset: number) => (offset % 7 === 0 ? "limit = 400" : match)) + const file = "src/review/large-store.ts" + + const fixture = await setupTimelineBenchmark(page, { + historyTurns: 2, + eventBatch: 1, + vcsDiff: [{ file, patch: fullPatch(file, before, after), additions: 1, deletions: 1, status: "modified" }], + }) + + fixture.transport.enqueue(textEvents(["Ready"])) + await expect(fixture.text).toContainText("Ready") + await fixture.waitForStableGeometry() + + await page.getByRole("button", { name: "Toggle review" }).click() + await expect(page.locator("#review-panel")).toBeVisible() + await browser.startTracing(page, { categories: traceCategories }) + const started = Date.now() + await openReviewLauncher(page.locator("#review-panel")) + await page + .waitForFunction( + () => + [...document.querySelectorAll('#review-panel [data-component="file"][data-mode="diff"]')].some( + (viewer) => + !!viewer + .querySelector("diffs-container") + ?.shadowRoot?.querySelector('[data-line] span[style*="--syntax-keyword"]'), + ), + undefined, + { timeout: 300_000, polling: "raf" }, + ) + .catch(async (error) => { + console.log("review pane", JSON.stringify(await describeReview(page))) + throw error + }) + const highlightedMs = Date.now() - started + // Let the pool finish the rest of the file so its time lands in the trace. + await page.waitForTimeout(3_000) + const trace = summarizeTrace(await browser.stopTracing()) + const memory = await measureMemory(page, browser) + report( + { highlightedMs, ...trace, ...memory, highlightsLoaded: loaded.highlights() }, + { engine, diffChars: before.length, diffLines: before.split("\n").length }, + ) + }) +} + +benchmark.describe("stills", () => { + benchmark.skip(!process.env.HIGHLIGHTER_STILLS, "set HIGHLIGHTER_STILLS to a directory") + + for (const engine of engines) { + for (const scheme of ["light", "dark"] as const) { + benchmark(`code in ${scheme} with ${engine}`, async ({ page, report }) => { + benchmark.setTimeout(420_000) + const directory = process.env.HIGHLIGHTER_STILLS! + await mkdir(directory, { recursive: true }) + await useEngine(page, engine) + await page.addInitScript((scheme) => localStorage.setItem("opencode-color-scheme", scheme), scheme) + const marker = `stills-complete-${engine}` + const before = samples[0].code + const after = before.replace("limit = 200", "limit = 400").replace('"Untitled"', '"New session"') + const file = "src/store.ts" + + const fixture = await setupTimelineBenchmark(page, { + historyTurns: 0, + eventBatch: 10_000, + vcsDiff: [{ file, patch: fullPatch(file, before, after), additions: 2, deletions: 2, status: "modified" }], + }) + + await page.setViewportSize({ width: 1500, height: 1800 }) + fixture.transport.enqueue(textEvents(["Stills", ...chunk(sampleReply(1, marker), 400)])) + await expect(fixture.text).toContainText(marker, { timeout: 60_000 }) + await fixture.waitForStableGeometry() + + const blocks = fixture.text.locator('[data-component="markdown-code"]') + await expect(blocks).toHaveCount(samples.length) + const tokens: Record = {} + + for (const [index, sample] of samples.entries()) { + const block = blocks.nth(index) + await block.scrollIntoViewIfNeeded() + await page.waitForTimeout(150) + await block.screenshot({ path: path.join(directory, `${engine}-${scheme}-${sample.lang}.png`) }) + tokens[sample.lang] = await block.evaluate((root) => + [...root.querySelectorAll("code span")].flatMap((span): [string, string, string, string][] => { + if (span.children.length > 0) return [] + const style = getComputedStyle(span) + + return [[span.textContent ?? "", style.color, style.fontStyle, style.fontWeight]] + }), + ) + } + + await page.getByRole("button", { name: "Toggle review" }).click() + await openReviewLauncher(page.locator("#review-panel")) + const viewer = page.locator('#review-panel [data-component="file"][data-mode="diff"]').first() + await expect + .poll(() => + viewer.evaluate( + (element) => + !!element + .querySelector("diffs-container") + ?.shadowRoot?.querySelector('span[style*="--syntax-keyword"]'), + ), + ) + .toBe(true) + .catch(async (error) => { + console.log("review pane", JSON.stringify(await describeReview(page))) + throw error + }) + await page.waitForTimeout(500) + await page.locator("#review-panel").screenshot({ path: path.join(directory, `${engine}-${scheme}-review.png`) }) + tokens.review = await viewer.evaluate((element) => + [ + ...(element + .querySelector("diffs-container") + ?.shadowRoot?.querySelectorAll('[data-line] span[style*="color"]') ?? []), + ] + .slice(0, 400) + .map((span) => { + const style = getComputedStyle(span) + + return [span.textContent ?? "", style.color, style.fontStyle, style.fontWeight] + }), + ) + await writeFile(path.join(directory, `${engine}-${scheme}-tokens.json`), JSON.stringify(tokens)) + report({ blocks: samples.length }, { engine, scheme }) + }) + } + } +}) + +function describeReview(page: Page) { + return page.evaluate(() => { + const panel = document.querySelector("#review-panel") + const viewers = [...(panel?.querySelectorAll('[data-component="file"]') ?? [])] + const shadow = panel?.querySelector("diffs-container")?.shadowRoot + + return { + text: panel?.textContent?.slice(0, 300), + viewers: viewers.map((viewer) => viewer.dataset.mode), + containers: panel?.querySelectorAll("diffs-container").length, + lines: shadow?.querySelectorAll("[data-line]").length, + spans: [...(shadow?.querySelectorAll("[data-line] span") ?? [])].slice(0, 6).map((span) => span.outerHTML), + html: panel?.innerHTML.slice(0, 1500), + } + }) +} + +const traceCategories = ["devtools.timeline", "toplevel", "disabled-by-default-devtools.timeline", "v8.execute"] + +async function useEngine(page: Page, engine: Engine) { + // Knobs persist the option's index: Shiki is 0, Highlights is 1. + await page.addInitScript( + (value) => localStorage.setItem("opencode.motion.v2", JSON.stringify(value ? { "code.highlighter": value } : {})), + engines.indexOf(engine), + ) + const urls: string[] = [] + page.context().on("request", (request) => urls.push(request.url())) + + return { highlights: () => urls.some((url) => /highlights/i.test(url)) } +} + +function textEvents(deltas: string[]) { + return [ + event("session.text.started", { sessionID, assistantMessageID, ordinal: 0 }, true), + ...deltas.map((delta) => textEvent(delta)), + ] +} + +function textEvent(delta: string) { + return event("session.text.delta", { sessionID, assistantMessageID, ordinal: 0, delta }) +} + +function event( + type: "session.text.started" | "session.text.delta", + data: { sessionID: string; assistantMessageID: string; ordinal: number; delta?: string }, + durable = false, +) { + sequence++ + + const value = { + id: `evt_highlighter_${sequence}`, + created: 1700000002000 + sequence, + type, + data, + location: { directory: benchmarkLocation.directory }, + durable: durable ? { aggregateID: sessionID, seq: sequence, version: 1 } : undefined, + } + + // SAFETY: the wire shape of a text event as the timeline fixture sends it; the mock server forwards it verbatim. + // oxlint-disable-next-line anti-slop/no-chained-type-assertions -- see SAFETY above + return value as unknown as OpenCodeEvent +} + +function chunk(text: string, size: number) { + return Array.from({ length: Math.ceil(text.length / size) }, (_, index) => + text.slice(index * size, (index + 1) * size), + ) +} + +function fullPatch(file: string, before: string, after: string) { + const left = before.split("\n") + const right = after.split("\n") + + return [ + `diff --git a/${file} b/${file}`, + `--- a/${file}`, + `+++ b/${file}`, + `@@ -1,${left.length} +1,${right.length} @@`, + ...left.flatMap((line, index) => (line === right[index] ? [` ${line}`] : [`-${line}`, `+${right[index]}`])), + ].join("\n") +} + +type TraceEvent = { + ph: string + name: string + pid: number + tid: number + ts: number + dur?: number + args?: { name?: string } +} + +// Busy time per thread from top-level tasks: the renderer main thread, and every dedicated worker in that renderer. +function summarizeTrace(buffer: Buffer) { + // SAFETY: Chrome's trace format, which `stopTracing` returns as JSON. + const parsed = JSON.parse(buffer.toString()) as { traceEvents: TraceEvent[] } | TraceEvent[] + const events = Array.isArray(parsed) ? parsed : parsed.traceEvents + + const names = new Map( + events + .filter((item) => item.ph === "M" && item.name === "thread_name") + .map((item) => [`${item.pid}:${item.tid}`, item.args?.name ?? ""]), + ) + + const tasks = events + .filter((item) => item.ph === "X" && /RunTask$/.test(item.name) && item.dur) + .toSorted((a, b) => a.ts - b.ts) + + const threads = new Map() + const ends = new Map() + // Only outermost tasks: a RunTask nested inside another one is already part of its duration. + tasks.forEach((item) => { + const key = `${item.pid}:${item.tid}` + + if (item.ts < (ends.get(key) ?? 0)) return + ends.set(key, item.ts + item.dur!) + threads.set(key, [...(threads.get(key) ?? []), item.dur! / 1000]) + }) + + const main = [...threads.entries()] + .filter(([key]) => names.get(key) === "CrRendererMain") + .toSorted((a, b) => sum(b[1]) - sum(a[1]))[0] + + const pid = main?.[0].split(":")[0] + + const workers = [...threads.entries()].filter( + ([key]) => key.startsWith(`${pid}:`) && /DedicatedWorker/i.test(names.get(key) ?? ""), + ) + + const mainTasks = main?.[1] ?? [] + const workerTasks = workers.flatMap(([, durations]) => durations) + + return { + mainBusyMs: round(sum(mainTasks)), + mainLongTasks: mainTasks.filter((value) => value > 50).length, + mainLongTaskMs: round(sum(mainTasks.filter((value) => value > 50))), + mainBlockingMs: round(sum(mainTasks.map((value) => Math.max(0, value - 50)))), + mainMaxTaskMs: round(Math.max(0, ...mainTasks)), + workerBusyMs: round(sum(workerTasks)), + workerLongTasks: workerTasks.filter((value) => value > 50).length, + workerMaxTaskMs: round(Math.max(0, ...workerTasks)), + workerThreads: workers.length, + } +} + +// JS heap of the page and of each dedicated worker, including array buffers such as WebAssembly memory. +async function measureMemory(page: Page, browser: Browser) { + const cdp = await page.context().newCDPSession(page) + await cdp.send("HeapProfiler.collectGarbage") + const main: HeapUsage = await cdp.send("Runtime.getHeapUsage") + await cdp.detach() + const session = await browser.newBrowserCDPSession() + const targets = await session.send("Target.getTargets") + const pending = new Map void>() + session.on("Target.receivedMessageFromTarget", (message) => { + // SAFETY: a worker's CDP reply to one of the requests below. + const reply = JSON.parse(message.message) as { id: number; result?: HeapUsage } + pending.get(reply.id)?.(reply.result) + }) + + const workers = await Promise.all( + targets.targetInfos + .filter((target) => target.type === "worker") + .map(async (target, index) => { + const attached = await session.send("Target.attachToTarget", { targetId: target.targetId, flatten: false }) + + const call = (id: number, method: string) => + new Promise((resolve) => { + pending.set(id, resolve) + void session.send("Target.sendMessageToTarget", { + sessionId: attached.sessionId, + message: JSON.stringify({ id, method }), + }) + setTimeout(() => resolve(undefined), 5_000) + }) + + await call(index * 10 + 1, "HeapProfiler.collectGarbage") + + return call(index * 10 + 2, "Runtime.getHeapUsage") + }), + ) + + await session.detach() + const heap = (usage: HeapUsage | undefined) => (usage ? usage.usedSize + (usage.backingStorageSize ?? 0) : 0) + + return { + mainHeapMB: mb(heap(main)), + workerHeapMB: mb(workers.reduce((total, usage) => total + heap(usage), 0)), + workerHeaps: workers.map((usage) => mb(heap(usage))), + } +} + +type HeapUsage = { usedSize: number; totalSize: number; backingStorageSize?: number } + +function sum(values: number[]) { + return values.reduce((total, value) => total + value, 0) +} + +function round(value: number) { + return Math.round(value * 10) / 10 +} + +function mb(bytes: number) { + return Math.round((bytes / 1024 / 1024) * 10) / 10 +} diff --git a/packages/app/e2e/performance/highlighter/samples.ts b/packages/app/e2e/performance/highlighter/samples.ts new file mode 100644 index 000000000000..d55faed7c3e0 --- /dev/null +++ b/packages/app/e2e/performance/highlighter/samples.ts @@ -0,0 +1,270 @@ +// The same code in every language the highlighter comparison covers, in the order the stills show them. +export const samples = [ + { + lang: "ts", + code: `import { Effect, Schema } from "effect" +import type { Session } from "./session" + +/** A session row as the sidebar shows it. */ +export interface Row { + readonly id: string + readonly title: string + readonly updated: number +} + +const Title = Schema.String.pipe(Schema.minLength(1)) + +export class SessionStore { + #rows = new Map() + + constructor(private readonly limit = 200) {} + + upsert(session: Session): Row | undefined { + if (this.#rows.size >= this.limit) return undefined + const row = { id: session.id, title: session.title ?? "Untitled", updated: Date.now() } + this.#rows.set(row.id, row) + return row + } +} + +export const load = Effect.fn("load")(function* (id: string) { + const title = yield* Schema.decodeUnknown(Title)(\`session \${id}\`) + const pattern = /^ses_[a-z0-9]{26}$/i + return pattern.test(id) ? { id, title, ok: true as const } : null +}) +`, + }, + { + lang: "tsx", + code: `import { For, Show, createMemo, createSignal } from "solid-js" +import type { Row } from "./store" + +type Props = { rows: Row[]; onSelect?: (id: string) => void } + +export function SessionList(props: Props) { + const [query, setQuery] = createSignal("") + const visible = createMemo(() => props.rows.filter((row) => row.title.includes(query()))) + + return ( +
+ setQuery(event.currentTarget.value)} /> + 0} fallback={

No sessions

}> +
    + + {(row) => ( +
  • props.onSelect?.(row.id)} title={\`Updated \${row.updated}\`}> + {row.title} — {row.id.slice(0, 8)} +
    • + )} +     +
+ +
+ ) +} +`, + }, + { + lang: "python", + code: `from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from typing import Iterable + + +@dataclass(frozen=True) +class Row: + id: str + title: str + tags: list[str] = field(default_factory=list) + + +class SessionStore: + """Keeps the most recent sessions in memory.""" + + def __init__(self, limit: int = 200) -> None: + self._rows: dict[str, Row] = {} + self.limit = limit + + def upsert(self, row: Row) -> Row | None: + if len(self._rows) >= self.limit: + return None + self._rows[row.id] = row + return row + + +async def load(ids: Iterable[str]) -> list[Row]: + rows = [Row(id=i, title=f"session {i!r}", tags=["recent"]) for i in ids if i.startswith("ses_")] + await asyncio.sleep(0.01) # yield to the loop + return sorted(rows, key=lambda row: row.title) +`, + }, + { + lang: "go", + code: `package store + +import ( + "context" + "errors" + "fmt" + "sync" + "time" +) + +// Row is a session as the sidebar shows it. +type Row struct { + ID string \`json:"id"\` + Title string \`json:"title"\` + Updated time.Time \`json:"updated"\` +} + +var ErrFull = errors.New("store is full") + +type Store struct { + mu sync.Mutex + rows map[string]Row + limit int +} + +func (s *Store) Upsert(ctx context.Context, row Row) (Row, error) { + s.mu.Lock() + defer s.mu.Unlock() + if len(s.rows) >= s.limit { + return Row{}, fmt.Errorf("upsert %q: %w", row.ID, ErrFull) + } + s.rows[row.ID] = row + return row, ctx.Err() +} +`, + }, + { + lang: "css", + code: `@import url("./tokens.css"); + +:root { + --row-height: 32px; + --row-radius: 6px; +} + +.session-list > ul { + display: grid; + gap: 2px; + margin: 0; + padding: 0 4px; +} + +.session-list li[data-active="true"], +.session-list li:hover { + background-color: color-mix(in srgb, var(--surface-raised) 80%, transparent); + border-radius: var(--row-radius); +} + +#composer::placeholder { + color: rgba(120, 120, 128, 0.6); + font: 500 13px/20px "Inter", sans-serif !important; +} + +@media (prefers-reduced-motion: reduce) { + .session-list * { + transition: none; + } +} +`, + }, + { + lang: "json", + code: `{ + "$schema": "https://opencode.ai/config.json", + "model": "anthropic/claude-sonnet-4-5", + "theme": "opencode", + "autoupdate": true, + "share": null, + "limits": { "context": 200000, "output": 32000, "ratio": 0.75 }, + "agent": { + "build": { + "temperature": 0.2, + "tools": ["read", "edit", "bash"], + "permission": { "bash": "ask", "edit": "allow" } + } + }, + "mcp": { + "github": { + "type": "remote", + "url": "https://api.githubcopilot.com/mcp/", + "enabled": false + } + } +} +`, + }, + { + lang: "bash", + code: `#!/usr/bin/env bash +set -euo pipefail + +# Build every package that changed since the base branch. +base="\${1:-origin/beta}" +root="$(git rev-parse --show-toplevel)" + +changed=$(git diff --name-only "$base"... -- packages | cut -d/ -f2 | sort -u) + +for pkg in $changed; do + if [[ -f "$root/packages/$pkg/package.json" ]]; then + echo "building $pkg" + (cd "$root/packages/$pkg" && bun run build) || exit 1 + fi +done + +export NODE_ENV=production +count=$(echo "$changed" | wc -l | tr -d ' ') +printf 'built %d packages in %ss\\n' "$count" "$SECONDS" >&2 +`, + }, + { + lang: "diff", + code: `diff --git a/src/store.ts b/src/store.ts +index 3f2a1c0..9b8e7d4 100644 +--- a/src/store.ts ++++ b/src/store.ts +@@ -12,9 +12,11 @@ export class SessionStore { + #rows = new Map() + +- constructor(private readonly limit = 100) {} ++ constructor(private readonly limit = 200) {} + + upsert(session: Session): Row | undefined { +- if (this.#rows.size > this.limit) return ++ if (this.#rows.size >= this.limit) return undefined ++ // Keep the newest title. + const row = { id: session.id, title: session.title ?? "Untitled", updated: Date.now() } + this.#rows.set(row.id, row) + return row +`, + }, +] as const + +export function sampleReply(repeat: number, marker: string) { + const blocks = Array.from({ length: repeat }, (_, round) => + samples + .map( + (sample) => + `### ${sample.lang} ${round + 1}\n\nThe ${sample.lang} sample.\n\n\`\`\`${sample.lang}\n${sample.code}\`\`\``, + ) + .join("\n\n"), + ) + + return `\n\n## Code review\n\n${blocks.join("\n\n")}\n\nDone: ${marker}` +} + +/** A TypeScript file of about `bytes` characters, built from the TS sample with numbered identifiers. */ +export function largeSource(bytes: number) { + const unit = samples[0].code + const count = Math.floor(bytes / (unit.length + 40)) + + return Array.from( + { length: count }, + (_, index) => + `// section ${index}\n${unit.replaceAll("SessionStore", `SessionStore${index}`).replaceAll("load", `load${index}`)}`, + ).join("\n") +} diff --git a/packages/app/vite.pwa.ts b/packages/app/vite.pwa.ts index 20661ce107a2..72f28e019ad3 100644 --- a/packages/app/vite.pwa.ts +++ b/packages/app/vite.pwa.ts @@ -8,9 +8,13 @@ import { VitePWA } from "vite-plugin-pwa" const officeAsset = /(?:^|\/)(?:(?:docx_\w+|ooxml_opc|pptx_wasm|xlsx_wasm)_bg-[\w-]+\.wasm|residentEngineWorker-[\w-]+\.js|(?:Carlito|Caladea|LiberationSans|LiberationSerif|LiberationMono|NotoSansArabic|NotoNaskhArabic|NotoSansHebrew|Gelasio|ComicRelief|Inter|Roboto|SourceSans3|DMSans|DMSerifDisplay|OpenSans|Montserrat|Poppins|Oswald|Heebo)-[\w-]+\.ttf)$/ +// Pierre Highlights inlines its WebAssembly in these chunks (one per realm). It is an opt-in experiment behind +// Knobs, so the chunks load only when it is picked. +const highlightsAsset = /(?:^|\/)(?:markdown-)?highlights-[\w-]+\.js$/ + /** Whether a built file loads on demand instead of being precached. */ export function onDemandAsset(path: string) { - return officeAsset.test(path) + return officeAsset.test(path) || highlightsAsset.test(path) } export function serviceWorker(directory: string) { diff --git a/packages/session-ui/package.json b/packages/session-ui/package.json index c038f0c276f5..af5e893f8f74 100644 --- a/packages/session-ui/package.json +++ b/packages/session-ui/package.json @@ -70,6 +70,7 @@ "@opencode/ui": "workspace:*", "@opencode/util": "workspace:*", "@pierre/diffs": "catalog:", + "@pierre/highlights": "catalog:", "@shikijs/stream": "catalog:", "@solid-primitives/event-listener": "catalog:", "@solid-primitives/media": "catalog:", diff --git a/packages/session-ui/src/components/markdown-highlights.test.ts b/packages/session-ui/src/components/markdown-highlights.test.ts new file mode 100644 index 000000000000..1d579d891c79 --- /dev/null +++ b/packages/session-ui/src/components/markdown-highlights.test.ts @@ -0,0 +1,20 @@ +import { expect, test } from "bun:test" +import { createHighlightsStream, highlightsTokens, style } from "./markdown-highlights" + +test("streamed chunks finalize the same tokens as highlighting the whole block", () => { + const code = 'const a = `tpl ${1 + 2}\nnext` /* multi\nline */ + "s"\nfunction f(x: number) { return x }\n' + const stream = createHighlightsStream("ts") + const chunks = Array.from({ length: Math.ceil(code.length / 3) }, (_, index) => code.slice(index * 3, index * 3 + 3)) + const results = chunks.map((chunk) => stream.push(chunk)) + stream.dispose() + + const streamed = [...results.flatMap((result) => result.stable), ...results.at(-1)!.unstable].flatMap((token) => + token.content === "\n" || token.content.length === 0 ? [] : [[token.content, style(token)]], + ) + + const whole = highlightsTokens(code, "ts") + .flat() + .flatMap((token) => (token.content.length === 0 ? [] : [[token.content, style(token)]])) + + expect(streamed).toEqual(whole) +}) diff --git a/packages/session-ui/src/components/markdown-highlights.ts b/packages/session-ui/src/components/markdown-highlights.ts new file mode 100644 index 000000000000..521a8d09ac27 --- /dev/null +++ b/packages/session-ui/src/components/markdown-highlights.ts @@ -0,0 +1,69 @@ +import { LiveTokenizer, codeToTokens, isSupportedLanguage, type Lang, type ThemedToken } from "@pierre/highlights" +import { OpenCodeHighlightsTheme } from "@opencode/ui/context/marked-theme" + +// Pierre Highlights for markdown code blocks. Imported lazily by the markdown worker, since importing the package +// compiles its WebAssembly; languages it has no lexer for stay on Shiki. + +export const supportsLanguage = isSupportedLanguage + +export function highlightsTokens(code: string, lang: Lang) { + return codeToTokens(code, { lang, theme: OpenCodeHighlightsTheme }).tokens +} + +export function highlightsHtml(code: string, lang: Lang) { + const result = codeToTokens(code, { lang, theme: OpenCodeHighlightsTheme }) + + const lines = result.tokens.map( + (line) => + `${line.map((token) => `${text(token.content)}`).join("")}`, + ) + + return `
${lines.join("\n")}
` +} + +/** + * Streams one code block through an editable document: each delta is appended as an edit, so only the edited last line + * and the lines after it are lexed again, with lexer state carried across chunks. Lines that end before the last one are + * final; the last line is still growing. + */ +export function createHighlightsStream(lang: Lang) { + const live = new LiveTokenizer({ lang, theme: OpenCodeHighlightsTheme }) + let emitted = 0 + + return { + push(delta: string) { + const last = live.lineCount - 1 + const end = { line: last, character: live.getLineLength(last) } + live.applyEdits([{ range: { start: end, end }, newText: delta }]) + const tail = live.lineCount - 1 + + const stable = Array.from({ length: tail - emitted }, (_, index) => [ + ...live.getLineTokens(emitted + index).tokens, + { content: "\n", offset: 0 }, + ]).flat() + + emitted = tail + + return { stable, unstable: live.getLineTokens(tail).tokens } + }, + dispose: () => live.dispose(), + } +} + +export function style(token: Pick) { + return [ + token.color ? `color:${token.color}` : "", + token.fontStyle && token.fontStyle & 1 ? "font-style:italic" : "", + token.fontStyle && token.fontStyle & 2 ? "font-weight:bold" : "", + ] + .filter(Boolean) + .join(";") +} + +function text(value: string) { + return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">") +} + +function attribute(value: string) { + return text(value).replaceAll('"', """) +} diff --git a/packages/session-ui/src/components/markdown-worker-protocol.ts b/packages/session-ui/src/components/markdown-worker-protocol.ts index 03031009f091..53ef812f542f 100644 --- a/packages/session-ui/src/components/markdown-worker-protocol.ts +++ b/packages/session-ui/src/components/markdown-worker-protocol.ts @@ -1,11 +1,20 @@ import type { Projection } from "./markdown-stream" +import type { CodeHighlighter } from "../pierre/highlighter" export type MarkdownToken = [content: string, style: string] export type MarkdownWorkerRequest = - | { type: "parse"; id: number; text: string } + | { type: "parse"; id: number; text: string; highlighter?: CodeHighlighter } | { type: "project"; id: number; key: string; text: string; live: boolean } - | { type: "highlight"; id: number; key: string; text: string; language: string; complete?: boolean } + | { + type: "highlight" + id: number + key: string + text: string + language: string + complete?: boolean + highlighter?: CodeHighlighter + } | { type: "dispose"; key: string } export type MarkdownWorkerResponse = diff --git a/packages/session-ui/src/components/markdown-worker.ts b/packages/session-ui/src/components/markdown-worker.ts index 37c79d8097a2..2e3c75f7ba14 100644 --- a/packages/session-ui/src/components/markdown-worker.ts +++ b/packages/session-ui/src/components/markdown-worker.ts @@ -7,6 +7,7 @@ import { } from "./markdown-worker-protocol" import { createWorkerTransport } from "./markdown-worker-transport" import type { Projection } from "./markdown-stream" +import { codeHighlighter } from "../pierre/highlighter" type HighlightPending = { key: string @@ -79,7 +80,7 @@ export function parseMarkdown(text: string, signal: AbortSignal) { return new Promise((resolve, reject) => { parses.set(id, { resolve, reject }) signal.addEventListener("abort", abort, { once: true }) - instance.postMessage({ type: "parse", id, text } satisfies MarkdownWorkerRequest) + instance.postMessage({ type: "parse", id, text, highlighter: codeHighlighter() } satisfies MarkdownWorkerRequest) }).finally(() => { signal.removeEventListener("abort", abort) parses.delete(id) @@ -117,7 +118,7 @@ export function highlightStreamingCode(key: string, text: string, language: stri return new Promise((resolve, reject) => { pending.set(id, { key, complete, resolve, reject }) - transport.send({ type: "highlight", id, key, text, language, complete }) + transport.send({ type: "highlight", id, key, text, language, complete, highlighter: codeHighlighter() }) }) } diff --git a/packages/session-ui/src/components/markdown.worker.ts b/packages/session-ui/src/components/markdown.worker.ts index 32e4a9f2512b..fa2d367f6743 100644 --- a/packages/session-ui/src/components/markdown.worker.ts +++ b/packages/session-ui/src/components/markdown.worker.ts @@ -14,11 +14,21 @@ import { import type { MarkdownToken, MarkdownWorkerRequest, MarkdownWorkerResponse } from "./markdown-worker-protocol" import { createLatestWorkerQueue } from "./markdown-worker-queue" import { project, type Projection } from "./markdown-stream" +import type { Lang } from "@pierre/highlights" +import type { CodeHighlighter } from "../pierre/highlighter" + +type Highlights = typeof import("./markdown-highlights") + +type Tokenizer = { + push: (delta: string) => Promise<{ stable: MarkdownToken[]; unstable: MarkdownToken[] }> + dispose: () => void +} type Stream = { language: string + highlighter: CodeHighlighter source: string - tokenizer: ShikiStreamTokenizer + tokenizer: Tokenizer } const streams = new Map() @@ -27,10 +37,17 @@ const projections = new Map() let highlighter: ReturnType | undefined +// Undefined once Highlights failed to load or to highlight: this worker then stays on Shiki for the rest of the session, +// since a load failure (such as a blocked WebAssembly compile) would only repeat on every block. +let highlights: Promise | undefined + const highlightQueue = createLatestWorkerQueue>({ run: highlight, supersede: (request) => post({ type: "superseded", id: request.id, key: request.key }), - dispose: (key) => void streams.delete(key), + dispose: (key) => { + streams.get(key)?.tokenizer.dispose() + streams.delete(key) + }, }) const projectQueue = createLatestWorkerQueue>({ @@ -39,17 +56,33 @@ const projectQueue = createLatestWorkerQueue void projections.delete(key), }) -const parser = createMarkdownParser(async (code, language) => { +const shikiParser = createMarkdownParser(shikiHtml) + +const highlightsParser = createMarkdownParser(async (code, language) => { + const lexer = await getHighlights() + + if (!lexer?.supportsLanguage(language)) return shikiHtml(code, language) + + try { + return lexer.highlightsHtml(code, language) + } catch (error) { + disableHighlights(error) + + return shikiHtml(code, language) + } +}) + +async function shikiHtml(code: string, language: string) { const instance = await getHighlighter() - const name = language in bundledLanguages ? language : "text" + // SAFETY: `name` is a key of bundledLanguages or "text", which Shiki always has. + const name = (language in bundledLanguages ? language : "text") as BundledLanguage - if (!instance.getLoadedLanguages().includes(name)) - await instance.loadLanguage(bundledLanguages[name as BundledLanguage]) + if (!instance.getLoadedLanguages().includes(name)) await instance.loadLanguage(bundledLanguages[name]) return instance - .codeToHtml(code, { lang: name as BundledLanguage, theme: "OpenCode", tabindex: false }) + .codeToHtml(code, { lang: name, theme: "OpenCode", tabindex: false }) .replace("", ``) -}) +} self.onmessage = (event: MessageEvent) => { if (event.data.type === "dispose") { @@ -76,6 +109,7 @@ self.onmessage = (event: MessageEvent) => { async function parse(request: Extract) { try { + const parser = request.highlighter === "highlights" ? highlightsParser : shikiParser post({ type: "parse", id: request.id, html: await parser.parse(request.text) }) } catch (error) { post({ type: "error", id: request.id, message: error instanceof Error ? error.message : String(error) }) @@ -99,14 +133,31 @@ async function runProject(request: Extract) { try { + const lexer = request.highlighter === "highlights" ? await getHighlights() : undefined + const requested = request.language + + if (lexer?.supportsLanguage(requested)) { + const done = await highlightWithHighlights(lexer, request, requested).then( + () => true, + (error: unknown) => { + disableHighlights(error) + + return false + }, + ) + + if (done) return + } + const instance = await getHighlighter() - const language = request.language in bundledLanguages ? request.language : "text" + // SAFETY: `language` is a key of bundledLanguages or "text", which Shiki always has. + const language = (request.language in bundledLanguages ? request.language : "text") as BundledLanguage - if (!instance.getLoadedLanguages().includes(language)) - await instance.loadLanguage(bundledLanguages[language as BundledLanguage]) + if (!instance.getLoadedLanguages().includes(language)) await instance.loadLanguage(bundledLanguages[language]) if (request.complete) { - const result = instance.codeToTokens(request.text, { lang: language as BundledLanguage, theme: "OpenCode" }) + const result = instance.codeToTokens(request.text, { lang: language, theme: "OpenCode" }) + streams.get(request.key)?.tokenizer.dispose() streams.delete(request.key) post({ type: "highlight", @@ -125,28 +176,20 @@ async function highlight(request: Extract { + const tokenizer = new ShikiStreamTokenizer({ highlighter: instance, lang: language, theme: "OpenCode" }) - const stream = reset - ? { - language, - source: "", - tokenizer: new ShikiStreamTokenizer({ highlighter: instance, lang: language, theme: "OpenCode" }), - } - : previous + return { + push: async (delta) => { + const result = await tokenizer.enqueue(delta) - const result = await stream.tokenizer.enqueue(request.text.slice(stream.source.length)) - stream.source = request.text - streams.set(request.key, stream) - post({ - type: "highlight", - id: request.id, - key: request.key, - language, - reset, - stable: result.stable.filter((token) => token.content.length > 0).map(token), - unstable: result.unstable.filter((token) => token.content.length > 0).map(token), + return { + stable: result.stable.filter((token) => token.content.length > 0).map(token), + unstable: result.unstable.filter((token) => token.content.length > 0).map(token), + } + }, + dispose: () => undefined, + } }) } catch (error) { post({ @@ -158,6 +201,94 @@ async function highlight(request: Extract, + language: Lang, +) { + const convert = (value: { content: string; color?: string; fontStyle?: number }): MarkdownToken => [ + value.content, + lexer.style(value), + ] + + if (request.complete) { + const lines = lexer.highlightsTokens(request.text, language) + streams.get(request.key)?.tokenizer.dispose() + streams.delete(request.key) + post({ + type: "highlight", + id: request.id, + key: request.key, + language, + reset: true, + stable: lines + .flatMap((line, index) => (index === lines.length - 1 ? line : [...line, { content: "\n" }])) + .filter((value) => value.content.length > 0) + .map(convert), + unstable: [], + }) + + return + } + + await stream(request, language, "highlights", () => { + const tokenizer = lexer.createHighlightsStream(language) + + return { + push: async (delta) => { + const result = tokenizer.push(delta) + + return { + stable: result.stable.filter((value) => value.content.length > 0).map(convert), + unstable: result.unstable.filter((value) => value.content.length > 0).map(convert), + } + }, + dispose: tokenizer.dispose, + } + }) +} + +// Feeds the request's new suffix to the block's tokenizer, starting a fresh one when the text no longer extends what +// it has seen or the language or highlighter changed. +async function stream( + request: Extract, + language: string, + highlighter: CodeHighlighter, + create: () => Tokenizer, +) { + const previous = streams.get(request.key) + + const reset = + !previous || + previous.language !== language || + previous.highlighter !== highlighter || + !request.text.startsWith(previous.source) + + if (reset) { + previous?.tokenizer.dispose() + streams.delete(request.key) + } + + const current = reset ? { language, highlighter, source: "", tokenizer: create() } : previous + const result = await current.tokenizer.push(request.text.slice(current.source.length)) + current.source = request.text + streams.set(request.key, current) + post({ type: "highlight", id: request.id, key: request.key, language, reset, ...result }) +} + +function getHighlights() { + return (highlights ??= import("./markdown-highlights").catch((error: unknown) => { + disableHighlights(error) + + return undefined + })) +} + +function disableHighlights(error: unknown) { + highlights = Promise.resolve(undefined) + console.error("Pierre Highlights failed; markdown code uses Shiki for the rest of this session", error) +} + function getHighlighter() { return (highlighter ??= createHighlighter({ themes: [OpenCodeTheme], langs: [] })) } diff --git a/packages/session-ui/src/pierre/highlighter.ts b/packages/session-ui/src/pierre/highlighter.ts new file mode 100644 index 000000000000..2f33dd9948ae --- /dev/null +++ b/packages/session-ui/src/pierre/highlighter.ts @@ -0,0 +1,19 @@ +import { createSignal } from "solid-js" +import { motionTuning } from "@opencode/ui/motion-tuning" + +export type CodeHighlighter = "shiki" | "highlights" + +// Experiment: Pierre Highlights (WebAssembly lexers) in place of Shiki for review diffs and markdown code blocks. +const choice = motionTuning.choice("code.highlighter", { + label: "Code highlighter", + group: "Experiments", + value: "shiki", + labels: { shiki: "Shiki", highlights: "Highlights" }, +}) + +const [highlighter, setHighlighter] = createSignal(choice()) + +motionTuning.subscribe(() => setHighlighter(choice())) + +/** The highlighter picked in Knobs → Experiments. Reactive, so open diffs rebuild on the new pool when it changes. */ +export const codeHighlighter = highlighter diff --git a/packages/session-ui/src/pierre/worker.ts b/packages/session-ui/src/pierre/worker.ts index 0dbfb06d0060..065c29cd6925 100644 --- a/packages/session-ui/src/pierre/worker.ts +++ b/packages/session-ui/src/pierre/worker.ts @@ -1,6 +1,7 @@ import { WorkerPoolManager } from "@pierre/diffs/worker" import ShikiWorkerUrl from "@pierre/diffs/worker/worker.js?worker&url" import { registerOpenCodeTheme } from "@opencode/ui/context/marked-theme-register" +import { codeHighlighter, type CodeHighlighter } from "./highlighter" registerOpenCodeTheme() @@ -8,7 +9,7 @@ export function workerFactory(): Worker { return new Worker(ShikiWorkerUrl, { type: "module" }) } -function createPool(lineDiffType: "none" | "word-line") { +function createPool(lineDiffType: "none" | "word-line", highlighter: CodeHighlighter) { const pool = new WorkerPoolManager( { workerFactory, @@ -25,7 +26,7 @@ function createPool(lineDiffType: "none" | "word-line") { // Pierre renders with the pool's options, not the viewer's, whenever the pool works. The "none" pool only // serves diffs above the large-file threshold, so it carries the plain-text fallback the viewer requests. ...(lineDiffType === "none" && { maxLineDiffLength: 0, tokenizeMaxLineLength: 1 }), - preferredHighlighter: "shiki-wasm", + preferredHighlighter: highlighter === "highlights" ? "highlights" : "shiki-wasm", }, ) @@ -34,22 +35,23 @@ function createPool(lineDiffType: "none" | "word-line") { return pool } -let plain: WorkerPoolManager | undefined - -let diff: WorkerPoolManager | undefined +// One pool per diff mode and highlighter: a worker loads a single highlighter type, so switching highlighters +// moves viewers to another pool rather than reconfiguring this one. +const pools = new Map() export function getWorkerPool(lineDiffType: "none" | "word-line" = "word-line"): WorkerPoolManager | undefined { if (typeof window === "undefined") return - if (lineDiffType === "none") { - if (!plain) plain = createPool("none") + const highlighter = codeHighlighter() + const key = `${lineDiffType}:${highlighter}` + const existing = pools.get(key) - return plain - } + if (existing) return existing - if (!diff) diff = createPool("word-line") + const pool = createPool(lineDiffType, highlighter) + pools.set(key, pool) - return diff + return pool } export function getWorkerPools() { diff --git a/packages/ui/src/context/marked-theme-register.tsx b/packages/ui/src/context/marked-theme-register.tsx index 135381eab8d2..090b3ec94b2e 100644 --- a/packages/ui/src/context/marked-theme-register.tsx +++ b/packages/ui/src/context/marked-theme-register.tsx @@ -1,5 +1,5 @@ import { registerCustomTheme } from "@pierre/diffs" -import { OpenCodeTheme } from "./marked-theme" +import { OpenCodeHighlightsTheme, OpenCodeTheme } from "./marked-theme" let registered = false @@ -7,4 +7,5 @@ export function registerOpenCodeTheme() { if (registered) return registered = true registerCustomTheme("OpenCode", () => Promise.resolve(OpenCodeTheme)) + registerCustomTheme("OpenCode", () => Promise.resolve(OpenCodeHighlightsTheme), "zed") } diff --git a/packages/ui/src/context/marked-theme.tsx b/packages/ui/src/context/marked-theme.tsx index 527a35e678f0..e54576845c57 100644 --- a/packages/ui/src/context/marked-theme.tsx +++ b/packages/ui/src/context/marked-theme.tsx @@ -1,4 +1,4 @@ -import type { ThemeRegistrationResolved } from "@pierre/diffs" +import type { ThemeRegistration } from "shiki" export const OpenCodeTheme = { name: "OpenCode", @@ -369,4 +369,60 @@ export const OpenCodeTheme = { "variable.constant": "var(--syntax-constant)", "variable.defaultLibrary": "var(--syntax-unknown)", }, -} as unknown as ThemeRegistrationResolved +} satisfies ThemeRegistration + +// The same palette as a Zed theme for Pierre Highlights, which styles its own token categories instead of TextMate +// scopes. Each value names a CSS variable suffix, so `keyword: "syntax-keyword"` paints `var(--syntax-keyword)`. +export const OpenCodeHighlightsTheme = { + name: "OpenCode", + appearance: "dark", + cssVariables: { prefix: "--", defaults: { foreground: "var(--text-base)" } }, + style: { + "editor.background": "color-background-stronger", + "editor.foreground": "text-base", + syntax: { + comment: "syntax-comment", + hint: "syntax-comment", + attribute: "syntax-property", + boolean: "syntax-constant", + number: "syntax-constant", + constant: "syntax-constant", + label: "syntax-constant", + "string.escape": "syntax-constant", + "string.special.symbol": "syntax-constant", + "variable.special": "syntax-constant", + constructor: "syntax-type", + enum: "syntax-type", + namespace: "syntax-type", + type: "syntax-type", + "tag.component.jsx": "syntax-type", + "type.builtin": "syntax-primitive", + function: "syntax-primitive", + variant: "syntax-primitive", + "property.json_key": "syntax-primitive", + keyword: "syntax-keyword", + preproc: "syntax-keyword", + "keyword.operator": "syntax-operator", + operator: "syntax-operator", + string: "syntax-string", + tag: "syntax-string", + selector: "syntax-string", + "selector.class": "syntax-property", + "selector.id": "syntax-property", + "selector.pseudo": "syntax-property", + "string.regex": "syntax-regexp", + property: "syntax-property", + variable: "syntax-variable", + punctuation: "syntax-punctuation", + embedded: "syntax-punctuation", + "text.jsx": "syntax-punctuation", + title: { color: "syntax-info", font_weight: 700 }, + emphasis: { font_style: "italic" }, + "emphasis.strong": { color: "text-strong", font_weight: 700 }, + link_text: "syntax-unknown", + link_uri: "syntax-unknown", + "diff.plus": "syntax-diff-add", + "diff.minus": "syntax-diff-delete", + }, + }, +} diff --git a/packages/ui/src/context/marked.tsx b/packages/ui/src/context/marked.tsx index e06aabc8ec3c..42cb5686501f 100644 --- a/packages/ui/src/context/marked.tsx +++ b/packages/ui/src/context/marked.tsx @@ -1,5 +1,5 @@ import { getSharedHighlighter } from "@pierre/diffs" -import { bundledLanguages, type BundledLanguage } from "shiki" +import { bundledLanguages } from "shiki" import { createSimpleContext } from "./helper" import { createMarkdownParser } from "./marked-parser" import { registerOpenCodeTheme } from "./marked-theme-register" @@ -20,12 +20,8 @@ export const { use: useMarked, provider: MarkedProvider } = createSimpleContext( const name = language in bundledLanguages ? language : "text" - if (!highlighter.getLoadedLanguages().includes(name)) await highlighter.loadLanguage(name as BundledLanguage) + await highlighter.loadLanguages([name]) - return highlighter.codeToHtml(code, { - lang: name, - theme: "OpenCode", - tabindex: false, - }) + return highlighter.codeToHtml(code, { lang: name, theme: "OpenCode" }) }), })