Adding linting, running tests, and check links to the pipeline

.github/workflows/deploy.yml

@@ -9,7 +9,85 @@ on:
     branches:
       - main
 
+# Every job except close_pull_request_job carries the same `if`: run on pushes
+# to main and on open PRs, but only build the Crowdin `localization` branch
+# once a review is requested. GitHub Actions has no way to share a job
+# condition, so the expression is repeated verbatim; keep the copies in sync.
+#
+# Job layout:
+#   lint_job, doctest_job      independent checks on the repo sources
+#   build_job                  full multi-language site build
+#   link_check_job             needs the built site from build_job
+#   deploy_job                 waits for all of the above, so a failing check
+#                              blocks the deploy (production and PR previews)
 jobs:
+  lint_job:
+    if: (github.head_ref != 'localization' && github.event_name == 'push') || (github.head_ref != 'localization' && github.event_name == 'pull_request' && github.event.action != 'closed') || (github.head_ref == 'localization' && github.event.action == 'review_requested')
+    runs-on: ubuntu-latest
+    name: Lint build scripts
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Set up uv
+        # `./run scripts check` runs ruff and mypy through uvx.
+        uses: astral-sh/setup-uv@v10.0.1
+      - name: Install shell tooling
+        # shellcheck ships on the ubuntu runner image; shfmt does not, so it
+        # is pinned and downloaded (formatting rules can drift between shfmt
+        # versions, and a moving version would make the check flaky).
+        run: |
+          command -v shellcheck >/dev/null || sudo apt-get install -y shellcheck
+          curl -sSfL "https://github.com/mvdan/sh/releases/download/${SHFMT_VERSION}/shfmt_${SHFMT_VERSION}_linux_amd64" -o /usr/local/bin/shfmt
+          chmod +x /usr/local/bin/shfmt
+          shellcheck --version | head -2
+          shfmt --version
+        env:
+          SHFMT_VERSION: v3.14.0
+      - name: Check scripts
+        # ruff lint + format diff + mypy --strict on the qualified Python
+        # sources (pyproject.toml); shellcheck + shfmt diff on the run scripts.
+        run: ./run scripts check
+
+  doctest_job:
+    if: (github.head_ref != 'localization' && github.event_name == 'push') || (github.head_ref != 'localization' && github.event_name == 'pull_request' && github.event.action != 'closed') || (github.head_ref == 'localization' && github.event.action == 'review_requested')
+    runs-on: ubuntu-latest
+    name: Doctest C# code blocks
+    env:
+      # Optional repository secret: a direct download URL for the Linux x64
+      # Tabular Editor CLI archive (te-linux-x64.tar.gz). The CLI is gated
+      # behind sign-in, so there is no public URL to fetch it from. When the
+      # secret is unset, only the te-free annotation validation runs.
+      TE_CLI_URL: ${{ secrets.TE_CLI_DOWNLOAD_URL }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Validate doctest annotations
+        # Grammar and coverage of the ```csharp {compile} / {run ...} fences
+        # in every annotated content/*.md; no te needed.
+        run: ./run doctest validate
+      - name: Install Tabular Editor CLI
+        if: env.TE_CLI_URL != ''
+        run: |
+          mkdir -p "$RUNNER_TEMP/te"
+          curl -sSfL "$TE_CLI_URL" | tar -xz -C "$RUNNER_TEMP/te"
+          chmod +x "$RUNNER_TEMP/te/te"
+          echo "$RUNNER_TEMP/te" >> "$GITHUB_PATH"
+          "$RUNNER_TEMP/te/te" --version
+      - name: Run doctests
+        if: env.TE_CLI_URL != ''
+        # Compile and execute every annotated block against a throwaway model
+        # and diff its Output() against the documented **Output** fence.
+        run: ./run doctest
+      - name: Doctest execution skipped
+        if: env.TE_CLI_URL == ''
+        run: echo "::notice title=Doctests not executed::Set the TE_CLI_DOWNLOAD_URL repository secret to a te-linux-x64.tar.gz download URL to compile and run the annotated C# blocks in CI. Only annotation validation ran."
+
   build_job:
     if: (github.head_ref != 'localization' && github.event_name == 'push') || (github.head_ref != 'localization' && github.event_name == 'pull_request' && github.event.action != 'closed') || (github.head_ref == 'localization' && github.event.action == 'review_requested')
     runs-on: windows-latest
@@ -48,17 +126,73 @@ jobs:
           Write-Host "API files generated: $((Get-ChildItem content/api/*.yml).Count) YAML files"
         shell: pwsh
       - name: Build all documentation
-        run: python build-docs.py --all --skip-gen
+        # English DocFX warnings fail the build; es/zh warnings do not, so they
+        # are collected into a markdown report for the job summary below.
+        run: python build-docs.py --all --skip-gen --warnings-report build-warnings.md
+        env:
+          PYTHONUNBUFFERED: '1'  # keep the per-language banners in order with DocFX's own output in the log
+      - name: Publish DocFX warnings
+        if: always()
+        shell: bash
+        run: |
+          if [ -f build-warnings.md ]; then
+            cat build-warnings.md >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo '## DocFX build warnings' >> "$GITHUB_STEP_SUMMARY"
+            echo 'No report was produced (the build stopped before any language was built).' >> "$GITHUB_STEP_SUMMARY"
+          fi
       - name: Upload build artifact
         uses: actions/upload-artifact@v4
         with:
           name: site
           path: _site
           retention-days: 7
 
+  link_check_job:
+    needs: build_job
+    runs-on: ubuntu-latest
+    name: Check links
+    # A full run fetches every unique external URL once (a few thousand), so
+    # it takes minutes; the cap keeps a stalled host from holding the deploy.
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Download build artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: site
+          path: _site
+      - name: Check links
+        # Internal (own-site) links are errors and fail this job: local files,
+        # anchors, and old root-style paths (those are resolved against the
+        # live site's redirects). External links are warnings only: they are
+        # listed in the report and never fail the run, since third-party
+        # outages and bot-blocking are outside our control.
+        shell: bash
+        run: python build_scripts/check_links.py validate stats 2>&1 | tee link-check-report.txt
+      - name: Publish link report
+        # The report (including the external URLs to verify by hand) goes to
+        # the job summary, so warnings are visible without opening the log.
+        if: always()
+        shell: bash
+        run: |
+          {
+            echo '## Link check report'
+            echo
+            echo 'Internal broken links fail this job. External URLs listed under "WARNINGS" are informational; verify them by hand.'
+            echo
+            echo '```'
+            head -c 900000 link-check-report.txt
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
   deploy_job:
     if: (github.head_ref != 'localization' && github.event_name == 'push') || (github.head_ref != 'localization' && github.event_name == 'pull_request' && github.event.action != 'closed') || (github.head_ref == 'localization' && github.event.action == 'review_requested')
-    needs: build_job
+    needs: [build_job, lint_job, doctest_job, link_check_job]
     runs-on: ubuntu-latest
     name: Deploy to Azure
     steps:

README.md

@@ -69,6 +69,7 @@ swa start _site
 | `--skip-api` | Reuse existing API metadata in content/api, ~30-40% faster (local markdown iteration only, requires `--serve`/`--lang`; never for testing/CI/CD/releases) |
 | `--permissive` | Don't treat English DocFX warnings as build failures (for local iteration; full/CI builds stay strict) |
 | `--sync` | Sync English fallback for missing/outdated translations (for local dev) |
+| `--warnings-report PATH` | Write a markdown summary of the DocFX warnings per built language to `PATH` (CI appends it to the job summary); written even when the build fails |
 
 ## What the Build Script Does
 
@@ -83,6 +84,38 @@ swa start _site
 9. **Injects SEO tags** - Adds hreflang and canonical tags to HTML files
 10. **Generates SWA config** - Creates `staticwebapp.config.json` for Azure Static Web Apps routing
 
+# Continuous integration
+
+Every pull request against `main` (and every push to `main`) runs the workflow in [`.github/workflows/deploy.yml`](.github/workflows/deploy.yml).
+The deploy step waits for all checks, so a failing check blocks both the production deploy and the PR preview site.
+
+| Job | What it runs | Fails when |
+|-----|--------------|------------|
+| Lint build scripts | `./run scripts check`: ruff, ruff format, mypy `--strict` on the qualified Python sources (see `pyproject.toml`); shellcheck and shfmt on `run` and `build_scripts/run_scripts/*.sh` | any lint, type, or formatting finding |
+| Doctest C# code blocks | `./run doctest validate` always; `./run doctest` (compile, run, and compare `**Output**` blocks) when a te CLI is available, see below | malformed annotation; with te: compile error, runtime error, or output mismatch |
+| Build Documentation | `python build-docs.py --all --skip-gen --warnings-report build-warnings.md` on Windows, uploads `_site`; the per-language DocFX warning report goes to the job summary | DocFX build failure or English DocFX warnings |
+| Check links | `python build_scripts/check_links.py validate stats` against the built `_site` | any broken **internal** link: missing file, missing anchor, or an old root-style path that no longer redirects on the live site |
+| Deploy to Azure | Azure Static Web Apps upload | only after all of the above pass |
+
+**External links are warnings only.**
+The link check fetches every unique external URL once, but a bad status or a network error on a third-party site never fails the run:
+the site is outside our control, and outages, bot-blocking, and link rot should not stop docs from shipping.
+The full report, including the list of external URLs to verify by hand, is attached to the job summary of the "Check links" job.
+Fix or remove genuinely dead external links as part of normal maintenance.
+
+**Translation build warnings are visible but not blocking.**
+Only English DocFX warnings fail the build; the es and zh builds are Crowdin-managed translations and may carry warnings such as broken bookmarks or missing includes.
+The "Build Documentation" job summary lists every DocFX warning per language, so translation problems can be spotted and fed back without blocking a deploy.
+
+**Executing the doctests needs the te CLI.**
+The CLI download is gated behind sign-in, so the workflow cannot fetch it from a public URL.
+Set the repository secret `TE_CLI_DOWNLOAD_URL` to a direct download URL for `te-linux-x64.tar.gz` (for example a private blob with a SAS token);
+the doctest job then installs it and runs the full `./run doctest`.
+Without the secret, the job validates the annotations only and posts a notice.
+Use a te build aligned with the current Tabular Editor 3 release, otherwise the compare step reports false drift.
+
+Run the same checks locally before pushing: `./run scripts check`, `./run doctest`, and `./run build` followed by `./run check-links`.
+
 # Project Structure
 
 ```

build-docs.py

@@ -22,6 +22,9 @@
                     (~30-40% faster). LOCAL markdown iteration ONLY, with --serve/--lang;
                     never for testing, CI/CD, or release builds.
     --permissive    Don't fail the English build on DocFX warnings (local iteration)
+    --warnings-report PATH
+                    Write a markdown summary of the DocFX warnings per built language
+                    to PATH (for CI job summaries); written even when the build fails
 """
 
 import argparse
@@ -35,21 +38,58 @@
 
 
 _DOCFX_WARNING_RE = re.compile(r': warning ', re.IGNORECASE)
+_ANSI_RE = re.compile(r'\x1b\[[0-9;]*m')
 
+# Language -> DocFX warning diagnostics seen while building it. Filled by
+# build_language(); rendered by write_warnings_report() for --warnings-report.
+BUILD_WARNINGS: dict[str, list[str]] = {}
 
-def run_command(cmd: list[str], description: str, check: bool = True, fail_on_warnings: bool = False) -> int:
+
+def _use_utf8_stdio() -> None:
+    """Make stdout/stderr UTF-8 with replacement for unencodable characters.
+
+    DocFX echoes translated headings and file names (e.g. Chinese) in its
+    diagnostics, which this script re-prints while streaming. When stdout is
+    redirected on Windows (CI logs, `| tee`), Python defaults to the ANSI code
+    page and print() raises UnicodeEncodeError on them, aborting the build over
+    a log line. A real console is already UTF-16/UTF-8 on Windows, so this only
+    changes the redirected case.
+    """
+    for stream in (sys.stdout, sys.stderr):
+        if hasattr(stream, "reconfigure"):
+            stream.reconfigure(encoding="utf-8", errors="replace")
+
+
+def _clean_warning_line(line: str) -> str:
+    """Strip colour codes and the absolute working-directory prefix from a DocFX diagnostic."""
+    text = _ANSI_RE.sub('', line).rstrip()
+    cwd = str(Path.cwd())
+    return text.replace(cwd + os.sep, '').replace(cwd + '/', '')
+
+
+def run_command(
+    cmd: list[str],
+    description: str,
+    check: bool = True,
+    fail_on_warnings: bool = False,
+    warnings: list[str] | None = None,
+) -> int:
     """Run a command and return exit code.
 
     If fail_on_warnings=True, streams output line-by-line, counts DocFX warning
     diagnostics (lines matching ': warning '), and returns exit code 1 if any
     are found — even when the process itself exits 0.
+
+    If `warnings` is a list, output is streamed the same way and every warning
+    diagnostic is appended to it (colour codes and the working directory
+    stripped) for the --warnings-report summary; the exit code is unaffected.
     """
     print(f"\n{'='*60}")
     print(f"  {description}")
     print(f"{'='*60}")
     print(f"Running: {' '.join(cmd)}\n")
 
-    if fail_on_warnings:
+    if fail_on_warnings or warnings is not None:
         warning_count = 0
         process = subprocess.Popen(
             cmd,
@@ -64,13 +104,15 @@ def run_command(cmd: list[str], description: str, check: bool = True, fail_on_wa
             print(line, end='', flush=True)
             if _DOCFX_WARNING_RE.search(line):
                 warning_count += 1
+                if warnings is not None:
+                    warnings.append(_clean_warning_line(line))
         process.wait()
 
         if check and process.returncode != 0:
             print(f"Error: Command failed with exit code {process.returncode}")
             return process.returncode
 
-        if warning_count > 0:
+        if fail_on_warnings and warning_count > 0:
             print(f"\nError: DocFX produced {warning_count} warning(s). Failing build.")
             return 1
 
@@ -234,13 +276,49 @@ def build_language(lang: str, sync: bool = False, skip_api: bool = False, permis
     # `docfx build` skips API metadata regeneration and reuses the existing
     # content/api/*.yml (the _apiSource DLLs don't change between content edits),
     # which is ~30-40% faster; bare `docfx` regenerates metadata then builds.
+    BUILD_WARNINGS[lang] = []
     return run_docfx(
         [*(["build"] if skip_api else []), config_path],
         f"Building {lang} documentation",
-        fail_on_warnings=(lang == "en" and not permissive)
+        fail_on_warnings=(lang == "en" and not permissive),
+        warnings=BUILD_WARNINGS[lang],
     )
 
 
+_REPORT_MAX_LINES = 200  # per language; keeps the job summary well under GitHub's 1 MiB cap
+
+
+def write_warnings_report(path: Path) -> None:
+    """Write a markdown summary of the DocFX warnings collected per built language.
+
+    Meant for CI job summaries (append the file to $GITHUB_STEP_SUMMARY). English
+    warnings already fail the build, but the localized builds only warn, so this
+    is where translation warnings become visible without reading the whole log.
+    """
+    lines = ["## DocFX build warnings", ""]
+    if not BUILD_WARNINGS:
+        lines += ["No language build ran.", ""]
+    else:
+        lines += ["| Language | Warnings |", "|----------|---------:|"]
+        lines += [f"| {lang} | {len(found)} |" for lang, found in BUILD_WARNINGS.items()]
+        lines += [
+            "",
+            "English warnings fail the build. Other languages are translations managed outside "
+            "this repo, so their warnings are listed here but never block a deploy.",
+            "",
+        ]
+        for lang, found in BUILD_WARNINGS.items():
+            if not found:
+                continue
+            lines += [f"<details><summary>{lang}: {len(found)} warning(s)</summary>", "", "```text"]
+            lines += found[:_REPORT_MAX_LINES]
+            if len(found) > _REPORT_MAX_LINES:
+                lines.append(f"... {len(found) - _REPORT_MAX_LINES} more; see the build log")
+            lines += ["```", "", "</details>", ""]
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    print(f"Warnings report written to {path}")
+
+
 def copy_languages_manifest() -> int:
     """Copy languages.json to _site/ root for runtime access."""
     manifest_src = Path("metadata/languages.json")
@@ -335,6 +413,7 @@ def fix_xref_in_api() -> int:
 
 
 def main() -> int:
+    _use_utf8_stdio()
     parser = argparse.ArgumentParser(
         description="Build documentation for one or more languages",
         formatter_class=argparse.RawDescriptionHelpFormatter,
@@ -349,9 +428,20 @@ def main() -> int:
     parser.add_argument("--skip-api", action="store_true", help="LOCAL markdown iteration only (requires --serve/--lang): reuse existing content/api, ~30-40%% faster. NEVER for testing/CI/CD/releases")
     parser.add_argument("--permissive", action="store_true", help="Don't treat English DocFX warnings as build failures (for local iteration; keep full/CI builds strict)")
     parser.add_argument("--sync", action="store_true", help="Sync English fallback for missing/outdated translations (for local dev)")
-
+    parser.add_argument("--warnings-report", metavar="PATH", help="Write a markdown summary of DocFX warnings per built language to PATH (for CI job summaries); written even when the build fails")
+
     args = parser.parse_args()
-
+
+    try:
+        return _build(args)
+    finally:
+        # Written even after a failed language build, so CI still gets the partial picture.
+        if args.warnings_report:
+            write_warnings_report(Path(args.warnings_report))
+
+
+def _build(args: argparse.Namespace) -> int:
+    """Run the build described by the parsed command line; returns the exit code."""
     # List available languages
     if args.list:
         langs = get_available_languages()

build_scripts/README.md

@@ -42,6 +42,7 @@ Both need `te` on PATH.
 
 Current state only validates semantic bridge docs.
 The code block annotations can be used in any doc.
+CI runs `./run doctest validate` on every PR, and the full `./run doctest` when the `TE_CLI_DOWNLOAD_URL` repository secret provides a te CLI; see [Continuous integration](../README.md#continuous-integration).
 
 #### `te_script_runner.py` -- generic runner
 
@@ -181,6 +182,7 @@ Fragment/text failures keep their `#anchor` / `:~:text=` in that list (the bare
 the fragment is what broke); a wholly unreachable URL is listed bare.
 
 Exit codes: `1` if any internal (own-site) reference is broken, else `0`. External warnings never fail the run, so third-party link rot will not break CI.
+CI runs `validate stats` against the built site on every PR and attaches the report to the job summary; see [Continuous integration](../README.md#continuous-integration).
 
 Options to modify output: