From 9d626b052d347a290d56f8f1199ffa18024f65a5 Mon Sep 17 00:00:00 2001 From: Emre Sokullu Date: Tue, 22 Sep 2026 13:17:07 +0300 Subject: [PATCH 01/70] Fix nested Markdown code fence rendering --- package.json | 3 +- src/chrome/src/ui/history-text.js | 11 +- src/chrome/src/ui/markdown-render.js | 49 +++++++ src/chrome/src/ui/sidepanel.js | 4 +- src/chrome/src/ui/skill-markdown.js | 4 +- src/firefox/src/ui/history-text.js | 11 +- src/firefox/src/ui/markdown-render.js | 49 +++++++ src/firefox/src/ui/sidepanel.js | 4 +- src/firefox/src/ui/skill-markdown.js | 4 +- test/markdown-render.mjs | 197 ++++++++++++++++++++++++++ test/run.js | 2 +- 11 files changed, 324 insertions(+), 14 deletions(-) create mode 100644 test/markdown-render.mjs diff --git a/package.json b/package.json index 25511df7e..062324ebf 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,9 @@ "private": true, "type": "module", "scripts": { - "test": "npm run test:systemone && npm run test:systemone:fast && node --test test/browser-dialogs.mjs && npm run test:firefox-bidi && npm run test:runtime-lifecycle && npm run test:provider-limits && npm run test:accessibility-tree-benchmark && npm run test:toolbar-guard && npm run test:pdf-read && npm run test:pdf-selection && npm run test:social-contract && npm run test:safesocial && npm run test:build-unpacked && npm run test:attachment-drop && node test/run.js && node test/selection-scope-restoration.mjs && node scripts/benchmark-offline-relevance.mjs && npm run test:security", + "test": "npm run test:markdown && npm run test:systemone && npm run test:systemone:fast && node --test test/browser-dialogs.mjs && npm run test:firefox-bidi && npm run test:runtime-lifecycle && npm run test:provider-limits && npm run test:accessibility-tree-benchmark && npm run test:toolbar-guard && npm run test:pdf-read && npm run test:pdf-selection && npm run test:social-contract && npm run test:safesocial && npm run test:build-unpacked && npm run test:attachment-drop && node test/run.js && node test/selection-scope-restoration.mjs && node scripts/benchmark-offline-relevance.mjs && npm run test:security", "test:provider-limits": "node test/provider-model-limits.mjs", + "test:markdown": "node --test test/markdown-render.mjs", "test:accessibility-tree-benchmark": "node --test test/llm/lib/accessibility-tree-formats.test.mjs && node test/llm/accessibility-tree-benchmark.mjs --no-exact-tokenizer --check", "test:attachment-drop": "node test/attachment-drop.mjs", "test:security": "node test/security/injection-corpus.mjs", diff --git a/src/chrome/src/ui/history-text.js b/src/chrome/src/ui/history-text.js index 074daadec..b80a24369 100644 --- a/src/chrome/src/ui/history-text.js +++ b/src/chrome/src/ui/history-text.js @@ -125,10 +125,17 @@ export function historyTextFromElement(root, { markdown = true } = {}) { if (markdown && tagName === 'PRE') { const language = String(node.parentElement?.querySelector?.('.code-lang')?.textContent || '').trim(); ensureBreak(); - output += `\`\`\`${language}\n`; + const beforeCode = output; + output = ''; for (const child of Array.from(node.childNodes || [])) visit(child, false, true); ensureBreak(); - output += '```'; + const code = output; + // History must not reintroduce ambiguous fences around a Markdown + // document (or any code sample containing literal backtick runs). + let fenceLength = 3; + for (const match of code.matchAll(/`{3,}/g)) fenceLength = Math.max(fenceLength, match[0].length + 1); + const fence = '`'.repeat(fenceLength); + output = `${beforeCode}${fence}${language}\n${code}${fence}`; return; } if (markdown && tagName === 'CODE' && !inPre) { diff --git a/src/chrome/src/ui/markdown-render.js b/src/chrome/src/ui/markdown-render.js index f8aa7a878..f1d55e26f 100644 --- a/src/chrome/src/ui/markdown-render.js +++ b/src/chrome/src/ui/markdown-render.js @@ -57,6 +57,55 @@ export function codeFenceLanguage(infoString) { return String(infoString || '').trim().split(/\s+/, 1)[0] || ''; } +/** Replace whole fenced blocks, including an unfinished block during streaming. */ +export function replaceMarkdownCodeFences(value, renderBlock) { + const source = String(value ?? ''); + // Accept fences at the document root and inside the simple containers this + // renderer preserves (lists and blockquotes). The old unanchored matcher + // accepted these forms, while a root-only matcher mistakes their closer for + // a new opener and consumes the rest of the message as code. + const fenceLines = /^(?:(?: {0,3}>[ \t]?)*(?:(?:[ \t]*(?:[-+*]|\d+[.)])[ \t]+)?)[ \t]{0,3})(`{3,}|~{3,})([^\r\n]*)(?:\r?\n|$)/gm; + const output = []; + let cursor = 0; + let block = null; + const stack = []; + + for (const match of source.matchAll(fenceLines)) { + const [, fence, info] = match; + const validOpening = fence[0] !== '`' || !info.includes('`'); + const markdown = /^(?:md|markdown)$/i.test(codeFenceLanguage(info)); + if (!block) { + if (!validOpening) continue; + output.push(source.slice(cursor, match.index)); + block = { info, start: match.index + match[0].length }; + stack.push({ fence, markdown }); + continue; + } + + const active = stack[stack.length - 1]; + // A closing fence occupies its own line, has no info string, and is at + // least as long as its opener. Backticks inside source code are literal. + if (!info.trim() && fence[0] === active.fence[0] && fence.length >= active.fence.length) { + stack.pop(); + if (!stack.length) { + output.push(renderBlock(block.info, source.slice(block.start, match.index))); + // Leave the closing line's newline for the surrounding Markdown. + cursor = match.index + match[0].replace(/\r?\n$/, '').length; + block = null; + } + } else if (active.markdown && validOpening && info.trim() && fence === active.fence) { + // Models sometimes wrap a README in ```markdown and reuse ```lang + // inside it. Recover only this named Markdown nesting; ordinary code + // and correctly longer outer fences retain their literal contents. + stack.push({ fence, markdown }); + } + } + + if (block) output.push(renderBlock(block.info, source.slice(block.start))); + else output.push(source.slice(cursor)); + return output.join(''); +} + function tokenSpan(type, value) { const escaped = escapeCodeHtml(value); return type ? `${escaped}` : escaped; diff --git a/src/chrome/src/ui/sidepanel.js b/src/chrome/src/ui/sidepanel.js index bc6e8126d..5964754db 100644 --- a/src/chrome/src/ui/sidepanel.js +++ b/src/chrome/src/ui/sidepanel.js @@ -7,7 +7,7 @@ import { t, getLocale, setLocale, LANGUAGES, applyDOMTranslations, translationsForKey } from './i18n.js'; import { CAPABILITY_LABEL } from '../agent/permission-gate.js'; import { sanitizeMarkdownLinks } from './markdown-link.js'; -import { codeFenceLanguage, highlightCode, renderMarkdownHeadings, renderMarkdownTables } from './markdown-render.js'; +import { codeFenceLanguage, highlightCode, renderMarkdownHeadings, renderMarkdownTables, replaceMarkdownCodeFences } from './markdown-render.js'; import { applyMode, loadMode, watch } from './theme.js'; import { UI_SCALE_LEVELS, @@ -12829,7 +12829,7 @@ function formatMarkdown(text, options = {}) { // 1. Extract fenced code blocks BEFORE escaping HTML const codeBlocks = []; - text = text.replace(/```[ \t]*([^`\r\n]*)\r?\n([\s\S]*?)```/g, (_match, info, code) => { + text = replaceMarkdownCodeFences(text, (info, code) => { const lang = codeFenceLanguage(info); const id = `__CODEBLOCK_${codeBlocks.length}__`; codeBlocks.push({ lang: lang || '', code }); diff --git a/src/chrome/src/ui/skill-markdown.js b/src/chrome/src/ui/skill-markdown.js index 5e65cb0cf..2dd0f4ac6 100644 --- a/src/chrome/src/ui/skill-markdown.js +++ b/src/chrome/src/ui/skill-markdown.js @@ -5,7 +5,7 @@ import { escapeHtml } from './utils.js'; import { sanitizeMarkdownLinks } from './markdown-link.js'; -import { renderMarkdownHeadings, renderMarkdownTables } from './markdown-render.js'; +import { renderMarkdownHeadings, renderMarkdownTables, replaceMarkdownCodeFences } from './markdown-render.js'; function renderEmphasis(text) { return text @@ -38,7 +38,7 @@ function renderInlineMarkdown(value) { export function renderSkillMarkdown(content) { let text = String(content || ''); const codeBlocks = []; - text = text.replace(/```[ \t]*([^`\r\n]*)\r?\n([\s\S]*?)```/g, (_match, _info, code) => { + text = replaceMarkdownCodeFences(text, (_info, code) => { const placeholder = `__SKILL_CODE_BLOCK_${codeBlocks.length}__`; codeBlocks.push(code); return placeholder; diff --git a/src/firefox/src/ui/history-text.js b/src/firefox/src/ui/history-text.js index 074daadec..b80a24369 100644 --- a/src/firefox/src/ui/history-text.js +++ b/src/firefox/src/ui/history-text.js @@ -125,10 +125,17 @@ export function historyTextFromElement(root, { markdown = true } = {}) { if (markdown && tagName === 'PRE') { const language = String(node.parentElement?.querySelector?.('.code-lang')?.textContent || '').trim(); ensureBreak(); - output += `\`\`\`${language}\n`; + const beforeCode = output; + output = ''; for (const child of Array.from(node.childNodes || [])) visit(child, false, true); ensureBreak(); - output += '```'; + const code = output; + // History must not reintroduce ambiguous fences around a Markdown + // document (or any code sample containing literal backtick runs). + let fenceLength = 3; + for (const match of code.matchAll(/`{3,}/g)) fenceLength = Math.max(fenceLength, match[0].length + 1); + const fence = '`'.repeat(fenceLength); + output = `${beforeCode}${fence}${language}\n${code}${fence}`; return; } if (markdown && tagName === 'CODE' && !inPre) { diff --git a/src/firefox/src/ui/markdown-render.js b/src/firefox/src/ui/markdown-render.js index f8aa7a878..f1d55e26f 100644 --- a/src/firefox/src/ui/markdown-render.js +++ b/src/firefox/src/ui/markdown-render.js @@ -57,6 +57,55 @@ export function codeFenceLanguage(infoString) { return String(infoString || '').trim().split(/\s+/, 1)[0] || ''; } +/** Replace whole fenced blocks, including an unfinished block during streaming. */ +export function replaceMarkdownCodeFences(value, renderBlock) { + const source = String(value ?? ''); + // Accept fences at the document root and inside the simple containers this + // renderer preserves (lists and blockquotes). The old unanchored matcher + // accepted these forms, while a root-only matcher mistakes their closer for + // a new opener and consumes the rest of the message as code. + const fenceLines = /^(?:(?: {0,3}>[ \t]?)*(?:(?:[ \t]*(?:[-+*]|\d+[.)])[ \t]+)?)[ \t]{0,3})(`{3,}|~{3,})([^\r\n]*)(?:\r?\n|$)/gm; + const output = []; + let cursor = 0; + let block = null; + const stack = []; + + for (const match of source.matchAll(fenceLines)) { + const [, fence, info] = match; + const validOpening = fence[0] !== '`' || !info.includes('`'); + const markdown = /^(?:md|markdown)$/i.test(codeFenceLanguage(info)); + if (!block) { + if (!validOpening) continue; + output.push(source.slice(cursor, match.index)); + block = { info, start: match.index + match[0].length }; + stack.push({ fence, markdown }); + continue; + } + + const active = stack[stack.length - 1]; + // A closing fence occupies its own line, has no info string, and is at + // least as long as its opener. Backticks inside source code are literal. + if (!info.trim() && fence[0] === active.fence[0] && fence.length >= active.fence.length) { + stack.pop(); + if (!stack.length) { + output.push(renderBlock(block.info, source.slice(block.start, match.index))); + // Leave the closing line's newline for the surrounding Markdown. + cursor = match.index + match[0].replace(/\r?\n$/, '').length; + block = null; + } + } else if (active.markdown && validOpening && info.trim() && fence === active.fence) { + // Models sometimes wrap a README in ```markdown and reuse ```lang + // inside it. Recover only this named Markdown nesting; ordinary code + // and correctly longer outer fences retain their literal contents. + stack.push({ fence, markdown }); + } + } + + if (block) output.push(renderBlock(block.info, source.slice(block.start))); + else output.push(source.slice(cursor)); + return output.join(''); +} + function tokenSpan(type, value) { const escaped = escapeCodeHtml(value); return type ? `${escaped}` : escaped; diff --git a/src/firefox/src/ui/sidepanel.js b/src/firefox/src/ui/sidepanel.js index 12c8ec3e6..c6ed35f98 100644 --- a/src/firefox/src/ui/sidepanel.js +++ b/src/firefox/src/ui/sidepanel.js @@ -7,7 +7,7 @@ import { t, getLocale, setLocale, LANGUAGES, applyDOMTranslations, translationsForKey } from './i18n.js'; import { CAPABILITY_LABEL } from '../agent/permission-gate.js'; import { sanitizeMarkdownLinks } from './markdown-link.js'; -import { codeFenceLanguage, highlightCode, renderMarkdownHeadings, renderMarkdownTables } from './markdown-render.js'; +import { codeFenceLanguage, highlightCode, renderMarkdownHeadings, renderMarkdownTables, replaceMarkdownCodeFences } from './markdown-render.js'; import { applyMode, loadMode, watch } from './theme.js'; import { UI_SCALE_LEVELS, @@ -12416,7 +12416,7 @@ function formatMarkdown(text, options = {}) { // 1. Extract fenced code blocks BEFORE escaping HTML const codeBlocks = []; - text = text.replace(/```[ \t]*([^`\r\n]*)\r?\n([\s\S]*?)```/g, (_match, info, code) => { + text = replaceMarkdownCodeFences(text, (info, code) => { const lang = codeFenceLanguage(info); const id = `__CODEBLOCK_${codeBlocks.length}__`; codeBlocks.push({ lang: lang || '', code }); diff --git a/src/firefox/src/ui/skill-markdown.js b/src/firefox/src/ui/skill-markdown.js index 5e65cb0cf..2dd0f4ac6 100644 --- a/src/firefox/src/ui/skill-markdown.js +++ b/src/firefox/src/ui/skill-markdown.js @@ -5,7 +5,7 @@ import { escapeHtml } from './utils.js'; import { sanitizeMarkdownLinks } from './markdown-link.js'; -import { renderMarkdownHeadings, renderMarkdownTables } from './markdown-render.js'; +import { renderMarkdownHeadings, renderMarkdownTables, replaceMarkdownCodeFences } from './markdown-render.js'; function renderEmphasis(text) { return text @@ -38,7 +38,7 @@ function renderInlineMarkdown(value) { export function renderSkillMarkdown(content) { let text = String(content || ''); const codeBlocks = []; - text = text.replace(/```[ \t]*([^`\r\n]*)\r?\n([\s\S]*?)```/g, (_match, _info, code) => { + text = replaceMarkdownCodeFences(text, (_info, code) => { const placeholder = `__SKILL_CODE_BLOCK_${codeBlocks.length}__`; codeBlocks.push(code); return placeholder; diff --git a/test/markdown-render.mjs b/test/markdown-render.mjs new file mode 100644 index 000000000..bf366ecfa --- /dev/null +++ b/test/markdown-render.mjs @@ -0,0 +1,197 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import vm from 'node:vm'; +import { test } from 'node:test'; + +const readUi = (build, name) => fs.readFileSync(new URL(`../src/${build}/src/ui/${name}`, import.meta.url), 'utf8'); +const panelFormatter = (build) => { + const source = readUi(build, 'sidepanel.js'); + const start = source.indexOf('function formatMarkdown('); + assert.ok(start >= 0); + return source.slice(start, source.indexOf('\n}', start) + 2); +}; +// Reduced reproduction of a model-authored README with same-length nested +// fences. Keep user traces and their private page content out of the fixture. +const readme = [ + '# Example README', '', '## Role', + '```text', 'User request', ' |', ' v', 'Browser tools', '```', '', + '## Usage', '```javascript', 'const marker = "```";', + 'const html = "";', '```', '', + '### Parser notes', '```bash', 'node --test parser.test.mjs', '```', '', + '## License', 'See `LICENSE`.', '', +].join('\n'); +const draft = `Here is the draft:\n\n\`\`\`markdown\n${readme}\`\`\`\n\n## Next steps\nReview it.`; +const preContents = (html) => [...html.matchAll(/
([\s\S]*?)<\/code><\/pre>/g)].map(match => match[1]);
+
+for (const build of ['chrome', 'firefox']) {
+  const helpers = await import(`../src/${build}/src/ui/markdown-render.js`);
+  const { sanitizeMarkdownLinks } = await import(`../src/${build}/src/ui/markdown-link.js`);
+  const { escapeHtml } = await import(`../src/${build}/src/ui/utils.js`);
+  const { renderSkillMarkdown } = await import(`../src/${build}/src/ui/skill-markdown.js`);
+  const { historyTextFromElement } = await import(`../src/${build}/src/ui/history-text.js`);
+  const formatMarkdown = vm.runInNewContext(`(${panelFormatter(build)})`, {
+    ...helpers, sanitizeMarkdownLinks, escapeHtml, t: key => key,
+    scheduleMathRender() {}, setTimeout() {},
+  });
+
+  test(`${build}: nested README remains one complete, copyable Markdown block`, () => {
+    for (const language of ['markdown', 'md', 'MARKDOWN']) {
+      const source = draft.replace('```markdown', `\`\`\`${language}`);
+      for (const enhance of [true, false]) {
+        const html = formatMarkdown(source, { enhance });
+        assert.deepEqual(preContents(html), [helpers.escapeCodeHtml(readme)]);
+        assert.equal((html.match(/class="code-copy-btn"/g) || []).length, enhance ? 1 : 0);
+        assert.match(html, /
Next steps<\/h2>Review it\./);
+        assert.doesNotMatch(html, /