POC: add documentation delta reporting to PR checks

.github/workflows/on-pr.yml

@@ -43,3 +43,103 @@ jobs:
     with:
       bazel-target: "//:docs"
       tests-report-artifact: tests-report
+
+  docs-delta:
+    needs: [docs-build]
+    if: >-
+      github.event_name == 'pull_request' &&
+      github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-24.04
+    permissions:
+      actions: read
+      contents: read
+    steps:
+      - name: Check out pull request
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Check out published documentation history
+        continue-on-error: true
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          repository: ${{ github.repository }}
+          ref: gh-pages
+          path: .docs-baseline
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Download documentation artifact
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: github-pages
+          path: docs-artifact
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
+
+      - name: Generate documentation delta report
+        run: |
+          set -euo pipefail
+          uv run --locked --project tools/docs_delta docs-delta
+
+      - name: Upload documentation delta artifact
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: docs-delta
+          path: docs-artifact/docs-delta.md
+          if-no-files-found: error
+
+  docs-comment:
+    needs: [docs-build, docs-delta]
+    if: >-
+      always() &&
+      github.event_name == 'pull_request' &&
+      github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-24.04
+    permissions:
+      actions: read
+      pull-requests: write
+    steps:
+      - name: Download documentation delta artifact
+        if: needs.docs-delta.result == 'success'
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: docs-delta
+          path: docs-delta
+
+      - name: Prepare documentation preview comment
+        env:
+          DOCS_BUILD_RESULT: ${{ needs.docs-build.result }}
+          DOCS_DELTA_RESULT: ${{ needs.docs-delta.result }}
+        run: |
+          set -euo pipefail
+          if [ "$DOCS_BUILD_RESULT" = success ] && [ "$DOCS_DELTA_RESULT" = success ]; then
+            cat docs-delta/docs-delta.md > "$RUNNER_TEMP/docs-comment.md"
+          else
+            {
+              echo "Documentation preview for this pull request is unavailable."
+              echo
+              if [ "$DOCS_BUILD_RESULT" != success ]; then
+                echo "The documentation build did not complete successfully (result: ${DOCS_BUILD_RESULT})."
+              else
+                echo "The documentation delta job did not complete successfully (result: ${DOCS_DELTA_RESULT})."
+              fi
+            } > "$RUNNER_TEMP/docs-comment.md"
+          fi
+
+      - name: Find existing documentation preview comment
+        id: find-comment
+        uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: github-actions[bot]
+          body-includes: Documentation preview for this pull request
+
+      - name: Create or update documentation preview comment
+        uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-id: ${{ steps.find-comment.outputs.comment-id }}
+          body-path: ${{ runner.temp }}/docs-comment.md
+          edit-mode: replace

docs/internals/requirements/tool_verification.rst

@@ -82,7 +82,9 @@ Report record
    :post_template: tool_qualification_report
 
    Evaluates the S-CORE Docs-as-Code tool for building and checking
-   documentation and traceability data from RST/Markdown sources.
+   documentation and traceability data from RST/Markdown sources. It also
+   summarizes changed Needs between the published documentation and a pull
+   request's proposed documentation.
 
 Details
 -------

tools/BUILD

@@ -5,7 +5,7 @@
 # information regarding copyright ownership.
 #
 # This program and the accompanying materials are made available under the
-# terms of the Apache License, Version 2.0 which is available at
+# terms of the Apache License Version 2.0 which is available at
 # https://www.apache.org/licenses/LICENSE-2.0
 #
 # SPDX-License-Identifier: Apache-2.0
@@ -32,3 +32,19 @@ py_binary(
         "//src/helper_lib",
     ],
 )
+
+py_binary(
+    name = "docs_delta_cli",
+    srcs = ["docs_delta_main.py"] + glob(["docs_delta/*.py"]),
+    main = "docs_delta_main.py",
+    visibility = ["//visibility:public"],
+    deps = [],
+)
+
+# Retain the original Bazel label for callers while the package itself now
+# occupies the ``tools/docs_delta`` source path.
+alias(
+    name = "docs_delta",
+    actual = ":docs_delta_cli",
+    visibility = ["//visibility:public"],
+)

tools/docs_delta/README.md

@@ -0,0 +1,39 @@
+<!-- ----------------------------------------------------------------------------
+  Copyright (c) 2026 Contributors to the Eclipse Foundation
+
+  See the NOTICE file(s) distributed with this work for additional
+  information regarding copyright ownership.
+
+  This program and the accompanying materials are made available under the
+  terms of the Apache License Version 2.0 which is available at
+  https://www.apache.org/licenses/LICENSE-2.0
+
+  SPDX-License-Identifier: Apache-2.0
+----------------------------------------------------------------------------- -->
+
+# Documentation Delta
+
+Compare the Sphinx-Needs inventory and rendered HTML from two documentation
+builds, then write a Markdown report. The command uses only the Python standard
+library.
+
+Run the CLI from the repository root with:
+
+```sh
+uv run --locked --project tools/docs_delta docs-delta --help
+```
+
+For a local comparison, provide the baseline and current build directories and
+the URLs that reviewers should open:
+
+```sh
+uv run --locked --project tools/docs_delta docs-delta \
+  --baseline-dir /path/to/baseline \
+  --current-dir /path/to/current \
+  --base-url https://example.org/docs/main \
+  --pr-url https://example.org/docs/pr-123
+```
+
+In GitHub pull-request Actions, the CLI can resolve the baseline from a local,
+full-history `gh-pages` checkout and derive the preview URLs from Actions
+metadata. `docs-delta --help` lists the options for overriding those defaults.

tools/docs_delta/__init__.py

@@ -0,0 +1,50 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+"""Compare documentation builds and format a Markdown delta."""
+
+from .cli import main
+from .comparison import (
+    DocsDeltaError,
+    NeedChange,
+    NeedComparison,
+    NeedMap,
+    PageChange,
+    PageComparison,
+    compare_needs,
+    load_needs,
+)
+from .rendered_html import compare_html, normalize_html
+from .report import (
+    MAX_RENDERED_VALUE_LENGTH,
+    need_link,
+    render_report,
+    unavailable_report,
+)
+
+__all__ = [
+    "DocsDeltaError",
+    "MAX_RENDERED_VALUE_LENGTH",
+    "NeedChange",
+    "NeedComparison",
+    "NeedMap",
+    "PageChange",
+    "PageComparison",
+    "compare_html",
+    "compare_needs",
+    "load_needs",
+    "main",
+    "need_link",
+    "normalize_html",
+    "render_report",
+    "unavailable_report",
+]

tools/docs_delta/__main__.py

@@ -0,0 +1,17 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+"""Run the docs-delta command as ``python -m docs_delta``."""
+
+from .cli import main
+
+raise SystemExit(main())

tools/docs_delta/cli.py

@@ -0,0 +1,109 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+"""Command-line interface for creating documentation delta reports."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+from .comparison import DocsDeltaError, compare_needs, load_needs
+from .github import github_pull_request, resolve_urls, resolved_baseline, workspace_path
+from .rendered_html import compare_html
+from .report import render_report, unavailable_report, write_report
+
+
+def argument_parser() -> argparse.ArgumentParser:
+    """Describe the local and GitHub Actions forms of the command line."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--baseline-dir",
+        type=Path,
+        help="directory baseline; defaults to automatic gh-pages mode in PR Actions",
+    )
+    parser.add_argument(
+        "--baseline-mode",
+        choices=("directory", "gh-pages"),
+        help="baseline source; inferred when omitted",
+    )
+    parser.add_argument(
+        "--gh-pages-dir",
+        type=Path,
+        help="local full-history gh-pages checkout (defaults to .docs-baseline)",
+    )
+    parser.add_argument(
+        "--current-dir",
+        type=Path,
+        help="current documentation directory (defaults to docs-artifact)",
+    )
+    parser.add_argument("--base-url")
+    parser.add_argument("--pr-url")
+    parser.add_argument(
+        "--output",
+        type=Path,
+        help="report path (defaults to docs-artifact/docs-delta.md)",
+    )
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Run the CLI and write either a delta or an explicit unavailable report.
+
+    A baseline may legitimately be absent while publishing is in progress; in
+    that case the report is still successful and explains why it has no delta.
+    Missing or malformed current build data is an error because it would make
+    even the PR side of the comparison unreliable.
+    """
+
+    args = argument_parser().parse_args(argv)
+    try:
+        pull_request = github_pull_request(os.environ)
+        current_dir = args.current_dir or workspace_path(os.environ, "docs-artifact")
+        output = args.output or current_dir / "docs-delta.md"
+
+        with resolved_baseline(args, pull_request, os.environ) as (
+            baseline_dir,
+            baseline_reason,
+        ):
+            if baseline_dir is None or not baseline_dir.is_dir():
+                # A stale baseline would produce a plausible but misleading
+                # diff. Surface the missing comparison in the comment instead.
+                reason = baseline_reason or f"directory is missing: {baseline_dir}"
+                write_report(output, unavailable_report(reason))
+                return 0
+            try:
+                baseline_needs = load_needs(baseline_dir)
+            except DocsDeltaError as exc:
+                write_report(output, unavailable_report(str(exc)))
+                return 0
+
+            if not current_dir.is_dir():
+                raise DocsDeltaError(
+                    f"documentation directory is missing: {current_dir}"
+                )
+            current_needs = load_needs(current_dir)
+            base_url, pr_url = resolve_urls(args, pull_request, os.environ)
+            report = render_report(
+                compare_needs(baseline_needs, current_needs),
+                compare_html(baseline_dir, current_dir),
+                base_url=base_url,
+                pr_url=pr_url,
+            )
+            write_report(output, report)
+    except (DocsDeltaError, OSError) as exc:
+        print(f"docs_delta: error: {exc}", file=sys.stderr)
+        return 2
+    return 0