diff --git a/.gitignore b/.gitignore
index defcf74..6a0b1d5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,3 +23,7 @@ package-lock.json
# Non-Claude agent worktrees (created per CLAUDE.md isolation rules)
.worktrees/
+
+# Raw prettyhtml.com capture — third-party copyrighted JS/CSS/HTML, local reference only.
+# Clean-room behavior specs live in planning/*.md; the raw files never get committed.
+planning/captures/
diff --git a/CLAUDE.md b/CLAUDE.md
index 72d227a..585bbd1 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -63,7 +63,13 @@ Each tool lives in its own directory with a consistent pattern:
### Current Tools
-- **`/formatter/`** — HTML Formatter & Tidy. Pipeline: tokenize → indent (two-stage) → tidy → compress. Tidy dropdown is organized into three groups: **Formatting** (lowercase/sort/quote — our extras), **Cleaning (prettyhtml.com)** (the 10 options 1-for-1 with prettyhtml.com — Inline styles, Classes & IDs, Empty tags, Tags with 1 space, Successive spaces, Comments, Tag attributes, To plain text, AI Watermarks, Smart s; first 6 ON by default), and **Extras** (data-attrs, span-unwrap, strip-stray-line-breaks, etc.). Literal ports of `removeTagAttributes`, `aiWatermarkFixer`, and `smartNbsps` live in `formatter/app.mjs` (ES module). `normalizeStrayBreaks` (same module) runs as a pre-pass before `tidy()` to strip `
` residue between/around block elements and inside empty blocks (default-ON Extras option `opt-stray-breaks`), replicating prettyhtml.com's TinyMCE normalization layer without a DOM round-trip; tests in `formatter/tests/stray-breaks.test.mjs`. Tests in `formatter/tests/*.test.mjs` run via `npm test` (uses `linkedom` as DOMParser shim in Node). Options in `localStorage` key `htmlTidy_options`. See `planning/2026-05-27-prettyhtml-parity.md` for the algorithm snapshot / insurance documentation, and `planning/2026-06-24-stray-line-break-normalization-{design,plan}.md` for the stray-break feature.
+- **`/formatter/`** — HTML Formatter & Tidy. All logic lives in `formatter/app.mjs` (an ES module; there is no `app.js` here). Buttons: Indent (two-stage), Tidy, Compress.
+ - **Tidy runs through `runTidyPipeline(html, opts)`**, ordered to match prettyhtml.com's `convertText()`: stray-break normalization → whitespace pre-pass → script/style strip → to-plain-text (**first**, as theirs is) → nbsp collapse (option 5) → inter-tag gap joins → `tidy()` → nested-empty fixpoint → block-newline separation → looped whitespace post-pass → tag-attributes → AI-watermarks → smart-punctuation straightening → smart-nbsps → final cleanup. `replaceUntilStable()` mirrors their `helyettesit()` replace-to-idempotence semantics.
+ - **prettyhtml.com is two layers**: a TinyMCE DOM round-trip, then the string cleaners behind the ten checkboxes. We have no round-trip, so several default-ON **Extras** stand in for layer 1 — `opt-stray-breaks`, `opt-block-newlines`, `opt-nested-empties`, `opt-docs-residue`. Reasoning about their cleaners in isolation gives the wrong answer about what their site outputs; always check end-to-end.
+ - Dropdown groups: **Formatting** (lowercase/sort/quote), **Cleaning (prettyhtml.com)** (the 10 options 1-for-1; first 6 ON by default), **Extras** (block newlines, nested empties, Google Docs residue, script/style strip, straighten smart punctuation, stray line breaks, data-attrs, span-unwrap).
+ - **Deliberate divergences** (documented in `app.mjs` above the pipeline): E — options 1/2 parse attributes structurally rather than doing double-quote-only string surgery; G — empty-tag removal exempts `td/th/script/style/media` and requires matching tag names; H — one-space-tag removal accepts ` `; N — curly quotes are accepted as attribute delimiters, which is what makes HTML pasted out of Google Docs survive.
+ - Tests in `formatter/tests/*.test.mjs` via `npm test` (`linkedom` as DOMParser shim). `parity.test.mjs` runs the pipeline against `tests/fixtures/prettyhtml-golden.json` — black-box input/output pairs captured from the live site; a fixture with an `ours` field is a recorded deliberate divergence. Options persist in `localStorage` key `htmlTidy_options`.
+ - Docs: `planning/2026-09-03-prettyhtml-complete-capture.md` is the current clean-room spec (supersedes most of `planning/2026-05-27-prettyhtml-parity.md`); `planning/2026-06-24-stray-line-break-normalization-{design,plan}.md` covers stray breaks. The raw third-party capture lives in gitignored `planning/captures/` and must never be committed.
- **`/og-image/`** — OG Image Preview. Platform specs in `platforms.json`. Optional Cloudflare Worker CORS proxy in `functions/fetch-meta.js`. Fallback proxies for CORS.
### Suggested Tools System
diff --git a/README.md b/README.md
index 89dd27c..0d0c110 100644
--- a/README.md
+++ b/README.md
@@ -70,7 +70,8 @@ Beautify, format, and clean up messy or minified HTML instantly.
└── formatter/ # HTML Formatter & Tidy
├── index.html
├── styles.css
- └── app.js
+ ├── app.mjs # ES module — pipeline, cleaners, DOM wiring
+ └── tests/ # node --test, linkedom DOMParser shim
```
Each tool is self-contained in its own directory with its own `index.html`, making it easy to develop, test, and deploy independently.
diff --git a/eslint.config.js b/eslint.config.js
index 859875d..e09b808 100644
--- a/eslint.config.js
+++ b/eslint.config.js
@@ -41,6 +41,15 @@ module.exports = [
},
},
},
+ {
+ files: ['formatter/tests/**/*.mjs'],
+ languageOptions: {
+ sourceType: 'module',
+ globals: {
+ ...globals.node,
+ },
+ },
+ },
{
files: ['og-image/functions/**/*.js'],
languageOptions: {
@@ -51,6 +60,8 @@ module.exports = [
},
},
{
- ignores: ['node_modules/'],
+ // planning/captures/ holds the raw third-party prettyhtml.com capture. It is
+ // gitignored reference material, not our source — never lint or ship it.
+ ignores: ['node_modules/', 'planning/captures/'],
},
];
diff --git a/formatter/app.mjs b/formatter/app.mjs
index a930794..b90e39d 100644
--- a/formatter/app.mjs
+++ b/formatter/app.mjs
@@ -67,25 +67,66 @@ const BLOCK_ELEMENTS = new Set([
'td', 'th', 'dl', 'dt', 'dd', 'form', 'fieldset', 'address', 'pre',
]);
+/**
+ * Curly quotes accepted as attribute delimiters.
+ *
+ * Word processors (Google Docs especially) autocorrect straight quotes to curly
+ * ones even when the text is HTML *source*, so pasted markup routinely arrives as
+ * `class=“hero lede”`. Treating those as delimiters is a robustness fix, not a
+ * parity concern: prettyhtml.com's TinyMCE mis-parses the same input (it reads the
+ * value as unquoted and truncates at the first space) and only looks clean because
+ * its classes/IDs option then deletes the wreckage. Ours has no such backstop, so
+ * without this the invented attribute survives into the output.
+ *
+ * Each opener maps to the set of characters that may close it — a pair is accepted
+ * in either orientation, since autocorrect sometimes emits two openers or two
+ * closers when it guesses the word boundary wrong.
+ *
+ * Known limitation, deliberate: a value that mixes delimiter styles
+ * (`class=“hero lede"`) is NOT recognized and still falls to the bare-value path.
+ * Autocorrect converts both quotes of a pair, so this shape has not been observed
+ * in real input — and the obvious fix costs more than it buys. Letting a straight
+ * quote close a curly-opened value would truncate `alt=“He said "hi" to me”`, and
+ * letting a curly close a straight-opened one would truncate
+ * `alt="He said “hi” to me"` — quoted prose inside an attribute, which is both
+ * common and exactly what a word processor produces. Those two cases parse
+ * correctly today and are covered by tests; keep it that way unless a real
+ * mixed-delimiter sample turns up.
+ */
+const QUOTE_CLOSERS = {
+ '"': '"',
+ "'": "'",
+ '\u201C': '\u201D\u201C',
+ '\u201D': '\u201D\u201C',
+ '\u2018': '\u2019\u2018',
+ '\u2019': '\u2019\u2018',
+};
+
/**
* Parse an HTML attribute string into an array of {name, value, quote} objects.
*/
-function parseAttributes(attrString) {
+export function parseAttributes(attrString) {
const attrs = [];
if (!attrString || !attrString.trim()) return attrs;
- // Regex matches: name="value", name='value', name=value, or bare name
- const re = /([^\s=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|(\S+)))?/g;
+ // Regex matches: name="value", name='value', name=“value”, name=‘value’,
+ // name=value, or bare name. Curly-delimited values are normalized to straight
+ // quotes here so every downstream rebuild path emits valid HTML.
+ const re = /([^\s=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|[\u201C\u201D]([^\u201C\u201D]*)[\u201C\u201D]|[\u2018\u2019]([^\u2018\u2019]*)[\u2018\u2019]|(\S+)))?/g;
let match;
while ((match = re.exec(attrString)) !== null) {
const name = match[1];
const value = match[2] !== undefined ? match[2]
: match[3] !== undefined ? match[3]
: match[4] !== undefined ? match[4]
+ : match[5] !== undefined ? match[5]
+ : match[6] !== undefined ? match[6]
: null;
const quote = match[2] !== undefined ? '"'
: match[3] !== undefined ? "'"
- : match[4] !== undefined ? ''
+ : match[4] !== undefined ? '"'
+ : match[5] !== undefined ? "'"
+ : match[6] !== undefined ? ''
: null;
attrs.push({ name, value, quote });
}
@@ -96,7 +137,7 @@ function parseAttributes(attrString) {
* Tokenize an HTML string into an array of tokens.
* Each token has: { type, raw, tagName?, attributes?, content? }
*/
-function tokenize(html) {
+export function tokenize(html) {
const tokens = [];
let pos = 0;
const len = html.length;
@@ -224,9 +265,14 @@ function findTagEnd(html, pos) {
const len = html.length;
while (i < len) {
const ch = html[i];
- if (ch === '"' || ch === "'") {
- // Skip quoted attribute value
- const closeQuote = html.indexOf(ch, i + 1);
+ const closers = QUOTE_CLOSERS[ch];
+ if (closers) {
+ // Skip a quoted attribute value so a '>' inside it doesn't truncate the tag.
+ // Curly delimiters close on either member of their pair (see QUOTE_CLOSERS).
+ let closeQuote = -1;
+ for (let j = i + 1; j < len; j++) {
+ if (closers.includes(html[j])) { closeQuote = j; break; }
+ }
if (closeQuote === -1) return -1;
i = closeQuote + 1;
} else if (ch === '>') {
@@ -284,7 +330,7 @@ function rebuildTag(token) {
* @param {number} stage - 1 = block elements only, 2 = all elements
* @returns {string} Indented HTML
*/
-function indent(html, opts, stage) {
+export function indent(html, opts, stage) {
const tokens = tokenize(html);
const lines = [];
let indentLevel = 0;
@@ -406,6 +452,32 @@ function isBooleanAttr(name) {
return booleans.has(name.toLowerCase());
}
+/**
+ * True if an attribute is removed by the current options.
+ *
+ * Single source of truth for two call sites that must agree: buildTidyTag, which
+ * decides what to emit, and the unwrapSpans check, which decides whether a
+ * has anything left worth keeping. They were duplicated, and adding the Google
+ * Docs residue rules to one and not the other left a span whose only attribute
+ * was residue un-unwrapped while that same attribute got stripped. At shipped
+ * defaults the opt-nested-empties fixpoint hid it by catching the span on a
+ * second pass; with that Extra off, a stray survived.
+ */
+function isDroppedAttr(name, value, opts) {
+ const an = name.toLowerCase();
+ if (opts.removeStyles && (an === 'style' || an === 'valign' || an === 'align')) return true;
+ if (opts.removeClassesIds && (an === 'class' || an === 'id')) return true;
+ if (opts.removeDataAttrs && an.startsWith('data-')) return true;
+ if (opts.docsResidue) {
+ const lowerValue = (value || '').toLowerCase();
+ if (an === 'role' && lowerValue === 'presentation') return true;
+ if (an === 'aria-level') return true;
+ if (an === 'dir' && lowerValue === 'ltr') return true;
+ }
+ if (opts.removeEmptyAttrs && value === '' && !isBooleanAttr(an)) return true;
+ return false;
+}
+
/**
* Build a tidied tag string, applying cleaning options to attributes.
*/
@@ -417,11 +489,10 @@ function buildTidyTag(tagName, attrs, selfClose, opts) {
let value = attr.value;
let quote = attr.quote;
- const lowerAttrName = attrName.toLowerCase();
- if (opts.removeStyles && (lowerAttrName === 'style' || lowerAttrName === 'valign' || lowerAttrName === 'align')) return null;
- if (opts.removeClassesIds && (lowerAttrName === 'class' || lowerAttrName === 'id')) return null;
- if (opts.removeDataAttrs && lowerAttrName.startsWith('data-')) return null;
- if (opts.removeEmptyAttrs && value === '' && !isBooleanAttr(attrName)) return null;
+ // Google Docs residue is part of this: TinyMCE drops role/aria-level for
+ // prettyhtml.com at layer 1; we have no layer 1, so the option covers them
+ // explicitly. dir="ltr" survives on their site — deliberately better than parity.
+ if (isDroppedAttr(attrName, value, opts)) return null;
if (opts.quoteAttrs && value !== null && quote !== '"' && quote !== "'") {
quote = '"';
@@ -453,12 +524,16 @@ function buildTidyTag(tagName, attrs, selfClose, opts) {
* Tidy tokenized HTML — applies cleaning without changing whitespace.
* Returns { output, fixCount, tagCount }.
*/
-function tidy(html, opts) {
+export function tidy(html, opts) {
const tokens = tokenize(html);
const parts = [];
let fixCount = 0;
let tagCount = 0;
let unwrappedSpanDepth = 0;
+ // One entry per open : true if it was unwrapped as a Google Docs container.
+ // A plain stack rather than a depth counter, so a real nested keeps its
+ // own closing tag instead of consuming the wrapper's.
+ const boldStack = [];
for (let i = 0; i < tokens.length; i++) {
const token = tokens[i];
@@ -495,7 +570,7 @@ function tidy(html, opts) {
const canRemove = !VOID_ELEMENTS.has(lowerName) &&
!['script', 'style', 'iframe', 'canvas', 'video', 'audio', 'td', 'th'].includes(lowerName);
- // Case 1:
b" becomes "a b", not "ab"
+ // (divergence D). The doubled spaces this leaves are collapsed by the
+ // pipeline's post-pass.
+ t = t.replace(/<[^>]*>/g, ' ');
// Restore comments in order
t = t.replace(new RegExp(SENTINEL, 'g'), () => comments.shift());
return t;
@@ -939,6 +1034,18 @@ export function normalizeStrayBreaks(html) {
continue;
}
if (!hasDirectText[pIdx]) { i++; continue; } // empty block -> drop the
+
+ // A
with nothing but whitespace between it and its block's closing
+ // tag is filler, not a line break — it renders nothing. TinyMCE drops
+ // these for prettyhtml.com at layer 1; without that layer we do it here.
+ let k = i + 1;
+ while (k < n && tokens[k].type === TokenType.TEXT &&
+ parentBlock[k] === pIdx && tokens[k].content.trim() === '') k++;
+ if (k < n && isBlockClose(tokens[k]) &&
+ tokens[k].tagName.toLowerCase() === tokens[pIdx].tagName.toLowerCase()) {
+ i++;
+ continue;
+ }
// else: real line break inside text — keep it
}
out.push(t.raw);
@@ -947,6 +1054,265 @@ export function normalizeStrayBreaks(html) {
return out.join('');
}
+// ============================================================
+// TIDY PIPELINE — the order and scaffolding of prettyhtml.com convertText()
+//
+// Their engine is two layers. Layer 1 is a TinyMCE round-trip that normalizes
+// the DOM before any cleaner runs; layer 2 is the string cleaners behind the
+// ten checkboxes. We have no layer 1, so a few things TinyMCE does for them are
+// handled here by default-ON Extras options instead (normalizeStrayBreaks,
+// opt-nested-empties, opt-docs-residue). Reasoning about their cleaners in
+// isolation gives the wrong answer about what the site actually outputs — see
+// planning/2026-09-03-prettyhtml-complete-capture.md.
+//
+// Deliberate divergences, kept because ours is better:
+// E — options 1/2 parse attributes structurally; theirs does double-quote-only
+// string surgery, so style='x' or class=“x” slip past it.
+// G — canRemove exempts td/th/script/style/media; theirs deletes
/