From 3e8a10f9b3aa4ebc324cfe50d30d6a96159a9df6 Mon Sep 17 00:00:00 2001 From: DemchaAV Date: Thu, 1 Oct 2026 21:14:31 +0100 Subject: [PATCH] test(qa): report which presets each editor sets as the page does, and what keeps the rest short --- .github/workflows/ci.yml | 5 ++ CHANGELOG.md | 7 ++ CONTRIBUTING.md | 3 + .../fidelity/DocxFidelityCorpusTest.java | 1 + .../fidelity/DocxFidelityGateTest.java | 22 ++++++ .../fidelity/FidelityAcceptance.java | 77 +++++++++++++++++++ 6 files changed, 115 insertions(+) create mode 100644 qa/src/test/java/com/demcha/compose/document/templates/fidelity/FidelityAcceptance.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9c06fbdc..e01e6a402 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -597,6 +597,10 @@ jobs: -Dsurefire.failIfNoSpecifiedTests=false -Dgraphcompose.docxFidelity=libreoffice + - name: Show which presets LibreOffice sets as the page does + if: always() + run: cat qa/target/docx-fidelity/measured-libreoffice-acceptance.md >> "$GITHUB_STEP_SUMMARY" || true + - name: Upload the measurements if: always() uses: actions/upload-artifact@v7 @@ -604,6 +608,7 @@ jobs: name: docx-fidelity-${{ github.run_id }} path: | qa/target/docx-fidelity/measured-libreoffice*.tsv + qa/target/docx-fidelity/measured-libreoffice-acceptance.md qa/target/docx-fidelity/libreoffice.log if-no-files-found: warn # What a reviewer compares against the baseline, or takes as the new one diff --git a/CHANGELOG.md b/CHANGELOG.md index 28fcad470..7e301e7fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1489,6 +1489,13 @@ follow semantic versioning; release dates are ISO 8601. ### Tests +- **The DOCX fidelity corpus reports which presets are set as the page sets them.** Each run + writes `measured--acceptance.md` (`FidelityAcceptance`): a preset passes when the editor + sets it on the page's pages, finds 95% of its lines, sets them a median of 1pt or less from + the page, and no more than 10% past 2pt; the rest are listed with what keeps them short of + it, the furthest first. It reports and does not gate — the baseline gates. In Word 41 of the + 62 presets pass, in LibreOffice 37. CI writes LibreOffice's report into the job summary. + - **Word holds the DOCX export to its corpus too.** `scripts/docx-visual/word-fidelity.ps1` runs `DocxFidelityCorpusTest` in two halves around a conversion by a private Word instance: `export` writes the documents, Word converts them, and `word` measures Word's PDFs against diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7ab8bf7f5..5bea42bf1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -362,6 +362,9 @@ Choose the smallest tests that match the change: Windows with Word, `scripts/docx-visual/word-fidelity.ps1` (`-Update` to rewrite) exports the corpus, has a private Word instance convert it, and holds it to `word-windows.tsv` — run it from PowerShell (pwsh), after the install above, before a DOCX export change is opened. + Each run also writes `measured--acceptance.md`: which presets the editor sets as the + page does — the same pages, 95% of the lines found, a median of 1pt or less, no more than 10% + past 2pt — and what keeps the rest short of it. CI shows LibreOffice's in the job summary. If a change affects public docs, examples, or screenshots, update those assets in the same PR so the repository stays internally consistent. diff --git a/qa/src/test/java/com/demcha/compose/document/templates/fidelity/DocxFidelityCorpusTest.java b/qa/src/test/java/com/demcha/compose/document/templates/fidelity/DocxFidelityCorpusTest.java index e846db2a8..4e8960556 100644 --- a/qa/src/test/java/com/demcha/compose/document/templates/fidelity/DocxFidelityCorpusTest.java +++ b/qa/src/test/java/com/demcha/compose/document/templates/fidelity/DocxFidelityCorpusTest.java @@ -104,6 +104,7 @@ void everyDocumentStandsNoFurtherFromThePageThanItsBaseline() throws Exception { engine.resolve(document.stem() + ".pdf"), converted)); } FidelityBaseline.write(work.resolve("measured-" + mode + ".tsv"), note, measurements); + FidelityAcceptance.write(work.resolve("measured-" + mode + "-acceptance.md"), note, measurements); Path baseline = Path.of("src", "test", "resources", "docx-fidelity", mode + "-" + LibreOfficeConverter.platform() + ".tsv"); diff --git a/qa/src/test/java/com/demcha/compose/document/templates/fidelity/DocxFidelityGateTest.java b/qa/src/test/java/com/demcha/compose/document/templates/fidelity/DocxFidelityGateTest.java index b7f45c56e..96acae759 100644 --- a/qa/src/test/java/com/demcha/compose/document/templates/fidelity/DocxFidelityGateTest.java +++ b/qa/src/test/java/com/demcha/compose/document/templates/fidelity/DocxFidelityGateTest.java @@ -242,6 +242,28 @@ private static byte[] docx() throws Exception { } } + @Test + void aDocumentPassesWhenTheEditorSetsItAsThePageDoes() { + // engine pages, editor pages, lines, found, median, p90, past 2pt + assertThat(FidelityAcceptance.shortfalls(new FidelityMeasurement("cv-a", 1, 1, 100, 95, 1.0, 2, 9, Map.of()))) + .as("at every limit").isEmpty(); + assertThat(FidelityAcceptance.shortfalls(new FidelityMeasurement("cv-a", 1, 2, 100, 94, 1.01, 2, 10, Map.of()))) + .containsExactly("2 pages, not 1", "94 of 100 lines found", "median 1.01pt", "10 lines past 2pt"); + } + + @Test + void theAcceptanceReportNamesTheDocumentsThatFallShort() throws Exception { + Path file = dir.resolve("acceptance.md"); + FidelityAcceptance.write(file, "probe", List.of( + new FidelityMeasurement("cv-a", 1, 1, 100, 100, 0.2, 0.4, 0, Map.of()), + new FidelityMeasurement("cv-b", 1, 2, 100, 60, 3.0, 9, 50, Map.of()))); + + assertThat(String.join("\n", Files.readAllLines(file))) + .contains("1 of 2 documents pass") + .contains("| cv-b | 2 pages, not 1; 60 of 100 lines found; median 3.00pt; 50 lines past 2pt |") + .doesNotContain("| cv-a |"); + } + @Test void aMalformedBaselineRowIsRefusedByName() { assertThatThrownBy(() -> FidelityMeasurement.parse("cv-a\t1\tone\t40\t39\t0.25\t0.5\t1", Map.of())) diff --git a/qa/src/test/java/com/demcha/compose/document/templates/fidelity/FidelityAcceptance.java b/qa/src/test/java/com/demcha/compose/document/templates/fidelity/FidelityAcceptance.java new file mode 100644 index 000000000..5faee5e54 --- /dev/null +++ b/qa/src/test/java/com/demcha/compose/document/templates/fidelity/FidelityAcceptance.java @@ -0,0 +1,77 @@ +package com.demcha.compose.document.templates.fidelity; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; + +/** + * Whether an editor sets a document's DOCX as the page sets it, by four measures: the same + * number of pages, its lines found, set close, and few set far. + * + *

A document passes when the editor sets it on the page's number of pages, finds at least + * {@value #FOUND_SHARE_PERCENT} in a hundred of the page's lines, sets them a median of no more + * than {@value #MEDIAN_POINTS}pt from the page, and sets no more than + * {@value #FAR_SHARE_PERCENT} in a hundred more than 2pt off. It is a report of where each + * document stands, not a gate: the baseline ({@link FidelityBaseline}) is what keeps a document + * from moving further away.

+ */ +final class FidelityAcceptance { + + static final int FOUND_SHARE_PERCENT = 95; + static final double MEDIAN_POINTS = 1.0; + static final int FAR_SHARE_PERCENT = 10; + + private FidelityAcceptance() { + } + + /** What keeps a document from passing; empty when it passes. */ + static List shortfalls(FidelityMeasurement measured) { + List shortfalls = new ArrayList<>(); + if (measured.editorPages() != measured.enginePages()) { + shortfalls.add(measured.editorPages() + " pages, not " + measured.enginePages()); + } + if (measured.matched() * 100L < (long) FOUND_SHARE_PERCENT * measured.lines()) { + shortfalls.add(measured.matched() + " of " + measured.lines() + " lines found"); + } + if (measured.median() > MEDIAN_POINTS) { + shortfalls.add(String.format(Locale.ROOT, "median %.2fpt", measured.median())); + } + if (measured.over2() * 100L > (long) FAR_SHARE_PERCENT * Math.max(1, measured.matched())) { + shortfalls.add(measured.over2() + " lines past 2pt"); + } + return shortfalls; + } + + /** + * Writes the report: how many documents pass, then each document that does not, with what + * keeps it from passing, the furthest first. + */ + static void write(Path file, String note, Collection measurements) throws IOException { + List failing = measurements.stream() + .filter(measured -> !shortfalls(measured).isEmpty()) + .sorted(Comparator.comparingInt((FidelityMeasurement measured) -> measured.lines() - measured.matched() + + measured.over2()).reversed()) + .toList(); + List lines = new ArrayList<>(); + lines.add("# DOCX fidelity: " + note); + lines.add(""); + lines.add((measurements.size() - failing.size()) + " of " + measurements.size() + + " documents pass: the page's pages, " + FOUND_SHARE_PERCENT + "% of its lines found, a median of " + + MEDIAN_POINTS + "pt or less, no more than " + FAR_SHARE_PERCENT + "% past 2pt."); + if (!failing.isEmpty()) { + lines.add(""); + lines.add("| Document | Short of it |"); + lines.add("|---|---|"); + failing.forEach(measured -> lines.add("| " + measured.stem() + " | " + + String.join("; ", shortfalls(measured)) + " |")); + } + Files.createDirectories(file.toAbsolutePath().getParent()); + Files.write(file, lines, StandardCharsets.UTF_8); + } +}