From bcc785c7f6464af009f54ac958103b0db08a5f48 Mon Sep 17 00:00:00 2001 From: Nedas Date: Thu, 3 Sep 2026 14:58:42 +0200 Subject: [PATCH 1/4] Adding linting, running tests, and check links to the pipeline --- .github/workflows/deploy.yml | 122 +++++++++++++++++++++++- README.md | 28 ++++++ build_scripts/README.md | 2 + build_scripts/run_scripts/README.md | 2 +- content/features/dax-query.md | 4 +- content/how-tos/Master-model-pattern.md | 2 +- content/references/Roadmap2-h.md | 4 +- content/references/preferences.md | 2 +- 8 files changed, 158 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 81844d417..13b022927 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -9,7 +9,85 @@ on: branches: - main +# Every job except close_pull_request_job carries the same `if`: run on pushes +# to main and on open PRs, but only build the Crowdin `localization` branch +# once a review is requested. GitHub Actions has no way to share a job +# condition, so the expression is repeated verbatim; keep the copies in sync. +# +# Job layout: +# lint_job, doctest_job independent checks on the repo sources +# build_job full multi-language site build +# link_check_job needs the built site from build_job +# deploy_job waits for all of the above, so a failing check +# blocks the deploy (production and PR previews) jobs: + lint_job: + if: (github.head_ref != 'localization' && github.event_name == 'push') || (github.head_ref != 'localization' && github.event_name == 'pull_request' && github.event.action != 'closed') || (github.head_ref == 'localization' && github.event.action == 'review_requested') + runs-on: ubuntu-latest + name: Lint build scripts + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Set up uv + # `./run scripts check` runs ruff and mypy through uvx. + uses: astral-sh/setup-uv@v10 + - name: Install shell tooling + # shellcheck ships on the ubuntu runner image; shfmt does not, so it + # is pinned and downloaded (formatting rules can drift between shfmt + # versions, and a moving version would make the check flaky). + run: | + command -v shellcheck >/dev/null || sudo apt-get install -y shellcheck + curl -sSfL "https://github.com/mvdan/sh/releases/download/${SHFMT_VERSION}/shfmt_${SHFMT_VERSION}_linux_amd64" -o /usr/local/bin/shfmt + chmod +x /usr/local/bin/shfmt + shellcheck --version | head -2 + shfmt --version + env: + SHFMT_VERSION: v3.14.0 + - name: Check scripts + # ruff lint + format diff + mypy --strict on the qualified Python + # sources (pyproject.toml); shellcheck + shfmt diff on the run scripts. + run: ./run scripts check + + doctest_job: + if: (github.head_ref != 'localization' && github.event_name == 'push') || (github.head_ref != 'localization' && github.event_name == 'pull_request' && github.event.action != 'closed') || (github.head_ref == 'localization' && github.event.action == 'review_requested') + runs-on: ubuntu-latest + name: Doctest C# code blocks + env: + # Optional repository secret: a direct download URL for the Linux x64 + # Tabular Editor CLI archive (te-linux-x64.tar.gz). The CLI is gated + # behind sign-in, so there is no public URL to fetch it from. When the + # secret is unset, only the te-free annotation validation runs. + TE_CLI_URL: ${{ secrets.TE_CLI_DOWNLOAD_URL }} + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Validate doctest annotations + # Grammar and coverage of the ```csharp {compile} / {run ...} fences + # in every annotated content/*.md; no te needed. + run: ./run doctest validate + - name: Install Tabular Editor CLI + if: env.TE_CLI_URL != '' + run: | + mkdir -p "$RUNNER_TEMP/te" + curl -sSfL "$TE_CLI_URL" | tar -xz -C "$RUNNER_TEMP/te" + chmod +x "$RUNNER_TEMP/te/te" + echo "$RUNNER_TEMP/te" >> "$GITHUB_PATH" + "$RUNNER_TEMP/te/te" --version + - name: Run doctests + if: env.TE_CLI_URL != '' + # Compile and execute every annotated block against a throwaway model + # and diff its Output() against the documented **Output** fence. + run: ./run doctest + - name: Doctest execution skipped + if: env.TE_CLI_URL == '' + run: echo "::notice title=Doctests not executed::Set the TE_CLI_DOWNLOAD_URL repository secret to a te-linux-x64.tar.gz download URL to compile and run the annotated C# blocks in CI. Only annotation validation ran." + build_job: if: (github.head_ref != 'localization' && github.event_name == 'push') || (github.head_ref != 'localization' && github.event_name == 'pull_request' && github.event.action != 'closed') || (github.head_ref == 'localization' && github.event.action == 'review_requested') runs-on: windows-latest @@ -56,9 +134,51 @@ jobs: path: _site retention-days: 7 + link_check_job: + needs: build_job + runs-on: ubuntu-latest + name: Check links + # A full run fetches every unique external URL once (a few thousand), so + # it takes minutes; the cap keeps a stalled host from holding the deploy. + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Download build artifact + uses: actions/download-artifact@v4 + with: + name: site + path: _site + - name: Check links + # Internal (own-site) links are errors and fail this job: local files, + # anchors, and old root-style paths (those are resolved against the + # live site's redirects). External links are warnings only: they are + # listed in the report and never fail the run, since third-party + # outages and bot-blocking are outside our control. + shell: bash + run: python build_scripts/check_links.py validate stats 2>&1 | tee link-check-report.txt + - name: Publish link report + # The report (including the external URLs to verify by hand) goes to + # the job summary, so warnings are visible without opening the log. + if: always() + shell: bash + run: | + { + echo '## Link check report' + echo + echo 'Internal broken links fail this job. External URLs listed under "WARNINGS" are informational; verify them by hand.' + echo + echo '```' + head -c 900000 link-check-report.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + deploy_job: if: (github.head_ref != 'localization' && github.event_name == 'push') || (github.head_ref != 'localization' && github.event_name == 'pull_request' && github.event.action != 'closed') || (github.head_ref == 'localization' && github.event.action == 'review_requested') - needs: build_job + needs: [build_job, lint_job, doctest_job, link_check_job] runs-on: ubuntu-latest name: Deploy to Azure steps: diff --git a/README.md b/README.md index 6dc37306c..90964933a 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,34 @@ swa start _site 9. **Injects SEO tags** - Adds hreflang and canonical tags to HTML files 10. **Generates SWA config** - Creates `staticwebapp.config.json` for Azure Static Web Apps routing +# Continuous integration + +Every pull request against `main` (and every push to `main`) runs the workflow in [`.github/workflows/deploy.yml`](.github/workflows/deploy.yml). +The deploy step waits for all checks, so a failing check blocks both the production deploy and the PR preview site. + +| Job | What it runs | Fails when | +|-----|--------------|------------| +| Lint build scripts | `./run scripts check`: ruff, ruff format, mypy `--strict` on the qualified Python sources (see `pyproject.toml`); shellcheck and shfmt on `run` and `build_scripts/run_scripts/*.sh` | any lint, type, or formatting finding | +| Doctest C# code blocks | `./run doctest validate` always; `./run doctest` (compile, run, and compare `**Output**` blocks) when a te CLI is available, see below | malformed annotation; with te: compile error, runtime error, or output mismatch | +| Build Documentation | `python build-docs.py --all --skip-gen` on Windows, uploads `_site` | DocFX build failure or English DocFX warnings | +| Check links | `python build_scripts/check_links.py validate stats` against the built `_site` | any broken **internal** link: missing file, missing anchor, or an old root-style path that no longer redirects on the live site | +| Deploy to Azure | Azure Static Web Apps upload | only after all of the above pass | + +**External links are warnings only.** +The link check fetches every unique external URL once, but a bad status or a network error on a third-party site never fails the run: +the site is outside our control, and outages, bot-blocking, and link rot should not stop docs from shipping. +The full report, including the list of external URLs to verify by hand, is attached to the job summary of the "Check links" job. +Fix or remove genuinely dead external links as part of normal maintenance. + +**Executing the doctests needs the te CLI.** +The CLI download is gated behind sign-in, so the workflow cannot fetch it from a public URL. +Set the repository secret `TE_CLI_DOWNLOAD_URL` to a direct download URL for `te-linux-x64.tar.gz` (for example a private blob with a SAS token); +the doctest job then installs it and runs the full `./run doctest`. +Without the secret, the job validates the annotations only and posts a notice. +Use a te build aligned with the current Tabular Editor 3 release, otherwise the compare step reports false drift. + +Run the same checks locally before pushing: `./run scripts check`, `./run doctest`, and `./run build` followed by `./run check-links`. + # Project Structure ``` diff --git a/build_scripts/README.md b/build_scripts/README.md index bd0a01370..b99072b4d 100644 --- a/build_scripts/README.md +++ b/build_scripts/README.md @@ -42,6 +42,7 @@ Both need `te` on PATH. Current state only validates semantic bridge docs. The code block annotations can be used in any doc. +CI runs `./run doctest validate` on every PR, and the full `./run doctest` when the `TE_CLI_DOWNLOAD_URL` repository secret provides a te CLI; see [Continuous integration](../README.md#continuous-integration). #### `te_script_runner.py` -- generic runner @@ -181,6 +182,7 @@ Fragment/text failures keep their `#anchor` / `:~:text=` in that list (the bare the fragment is what broke); a wholly unreachable URL is listed bare. Exit codes: `1` if any internal (own-site) reference is broken, else `0`. External warnings never fail the run, so third-party link rot will not break CI. +CI runs `validate stats` against the built site on every PR and attaches the report to the job summary; see [Continuous integration](../README.md#continuous-integration). Options to modify output: diff --git a/build_scripts/run_scripts/README.md b/build_scripts/run_scripts/README.md index 6926f5a13..cab226672 100644 --- a/build_scripts/run_scripts/README.md +++ b/build_scripts/run_scripts/README.md @@ -143,7 +143,7 @@ and the authoring guidance below that section. `./run scripts` and its nested commands are the gateway to run scripts about run scripts. - Check the script-development tools are installed: `./run scripts setup` -- Lint and verify formatting, without modifying anything (the check a PR should pass): +- Lint and verify formatting, without modifying anything (CI runs this on every PR; see the [Continuous integration](../../README.md#continuous-integration) section of the project README): - `./run scripts check`: everything (all shell tooling, the qualified Python sources) - `./run scripts check file1.sh file2.py ...`: specific files, routed by kind - Apply the formatters (writes files): `./run scripts format [file ...]` diff --git a/content/features/dax-query.md b/content/features/dax-query.md index 78d58ec2f..eabd0508b 100644 --- a/content/features/dax-query.md +++ b/content/features/dax-query.md @@ -63,7 +63,7 @@ The "Apply" option syncs the DAX expression for all measures, columns or tables The "Apply Selection" and "Apply Selection & Sync" will only apply the measures, columns or tables within the current selection of the query editor. -Unlike the [DAX Script feature](xrefid:dax-scripts), only the expression property of a measure can be updated this way, as the DAX query syntax does not support specifying other properties, such as Description, Display Folder, etc. +Unlike the [DAX Script feature](xref:dax-scripts), only the expression property of a measure can be updated this way, as the DAX query syntax does not support specifying other properties, such as Description, Display Folder, etc. The "Apply" option has also been added to the right-click context menu. @@ -114,7 +114,7 @@ Customers ## Debugging DAX Query -DAX queries are one of the two places where it is possible to run the [DAX Debugger](xrefid:dax-debugger), the other being the Pivot Grid. +DAX queries are one of the two places where it is possible to run the [DAX Debugger](xref:dax-debugger), the other being the Pivot Grid. The DAX debugger unlocks the ability to understand how the DAX works inside a single cell. To start the debugger simply right click on the desired cell and choose 'Debug cell', which will start the debugger in the context of the chosen cell. diff --git a/content/how-tos/Master-model-pattern.md b/content/how-tos/Master-model-pattern.md index 4c6c7a092..666aa794b 100644 --- a/content/how-tos/Master-model-pattern.md +++ b/content/how-tos/Master-model-pattern.md @@ -284,7 +284,7 @@ start /wait /d "c:\Program Files (x86)\Tabular Editor" TabularEditor.exe Model.b This assumes that you are executing the command line within the directory of your Model.bim file (or Database.json file if using the "Save to Folder"-functionality). The -S switch instructs Tabular Editor to apply the supplied script to the model, and the -D switch performs the deployment. The -O switch allows overwriting an existing database with the same name, and the -R switch indicates that we also want to overwrite roles of the target database. ## Master model processing -If you have a dedicated processing server and large amounts of data overlap between the individual models, it may make sense for you to process the data into the master model first, before splitting it up. This way, you can avoid processing the same data several times, into individual models. **This assumes, however, that you are not processing any tables where the partition query has been changed between versions, as shown in [this section](/xref:Master-model-pattern#altering-partition-queries).** The recipe for this is outlined below: +If you have a dedicated processing server and large amounts of data overlap between the individual models, it may make sense for you to process the data into the master model first, before splitting it up. This way, you can avoid processing the same data several times, into individual models. **This assumes, however, that you are not processing any tables where the partition query has been changed between versions, as shown in [this section](#altering-partition-queries).** The recipe for this is outlined below: 1. (Optional - in case there were metadata changes) Deploy your master model to your processing server 2. Perform the processing you need on your master model (do not process tables that have version-specific partition queries). diff --git a/content/references/Roadmap2-h.md b/content/references/Roadmap2-h.md index 3fe1f070a..efd79f749 100644 --- a/content/references/Roadmap2-h.md +++ b/content/references/Roadmap2-h.md @@ -81,7 +81,7 @@ The layout and structure of the Model.bim file, makes it horrible for purposes o For better release management workflows with Tabular Models, it would be interesting if Tabular Editor could save/load a Model.bim file as a folder structure with individual files for measures, calculated columns, etc. There should be command-line options available for exporting/importing Model.bim files from/to this format, and it should be possible to deploy directly from this format (in cases where you don't need the Model.bim file itself). These individual files should contain the same JSON as the Model.bim file, but without the "ModifiedTime" information, so that they can easily be used in a version control system, allowing multiple developers to work on the same model at once. -**Update**: [Available in 2.2](/Advanced-features#folder-serialization). +**Update**: [Available in 2.2](xref:folder-serialization). **Update**: As of 2.3, options exist to store Perspective and Translation metadata as annotations on the individual objects. This is useful for source control scenarios with multiple developers, to avoid having single files that gets lots of edits when developers change translations, perspective memberships, etc. @@ -95,4 +95,4 @@ Today, it is already possible to connect Tabular Editor to a model hosted by Pow This is a standard feature in SSDT, which would be useful to have in Tabular Editor as well. -**Update**: [Available in 2.2](/Advanced-features#import-export-translations). +**Update**: [Available in 2.2](xref:import-export-translations). diff --git a/content/references/preferences.md b/content/references/preferences.md index fef7dda3f..cc5edb1be 100644 --- a/content/references/preferences.md +++ b/content/references/preferences.md @@ -830,4 +830,4 @@ List of addresses that should bypass the proxy (e.g., `localhost;*.company.local ## Next Steps -For a user-friendly guide to the most commonly adjusted preferences, see the getting started guide (Personalizing TE3)[xrefid: personalizing-te3]. +For a user-friendly guide to the most commonly adjusted preferences, see the getting started guide [Personalizing TE3](xref:personalizing-te3). From 50270793db602059e98416806d7d836a4f440f6a Mon Sep 17 00:00:00 2001 From: Nedas Date: Thu, 3 Sep 2026 15:00:04 +0200 Subject: [PATCH 2/4] Fixed versions --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 13b022927..9824f3a3a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -33,7 +33,7 @@ jobs: python-version: '3.11' - name: Set up uv # `./run scripts check` runs ruff and mypy through uvx. - uses: astral-sh/setup-uv@v10 + uses: astral-sh/setup-uv@v10.0.1 - name: Install shell tooling # shellcheck ships on the ubuntu runner image; shfmt does not, so it # is pinned and downloaded (formatting rules can drift between shfmt From 79767fa144842ddc45a7d39c89044c56e0ba9da2 Mon Sep 17 00:00:00 2001 From: Nedas Date: Fri, 4 Sep 2026 11:38:10 +0200 Subject: [PATCH 3/4] Adding warning to summary --- .github/workflows/deploy.yml | 16 ++++++- README.md | 7 ++- build-docs.py | 86 +++++++++++++++++++++++++++++++++--- 3 files changed, 101 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 9824f3a3a..bb0708bd1 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -126,7 +126,21 @@ jobs: Write-Host "API files generated: $((Get-ChildItem content/api/*.yml).Count) YAML files" shell: pwsh - name: Build all documentation - run: python build-docs.py --all --skip-gen + # English DocFX warnings fail the build; es/zh warnings do not, so they + # are collected into a markdown report for the job summary below. + run: python build-docs.py --all --skip-gen --warnings-report build-warnings.md + env: + PYTHONUNBUFFERED: '1' # keep the per-language banners in order with DocFX's own output in the log + - name: Publish DocFX warnings + if: always() + shell: bash + run: | + if [ -f build-warnings.md ]; then + cat build-warnings.md >> "$GITHUB_STEP_SUMMARY" + else + echo '## DocFX build warnings' >> "$GITHUB_STEP_SUMMARY" + echo 'No report was produced (the build stopped before any language was built).' >> "$GITHUB_STEP_SUMMARY" + fi - name: Upload build artifact uses: actions/upload-artifact@v4 with: diff --git a/README.md b/README.md index 90964933a..c77485545 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ swa start _site | `--skip-api` | Reuse existing API metadata in content/api, ~30-40% faster (local markdown iteration only, requires `--serve`/`--lang`; never for testing/CI/CD/releases) | | `--permissive` | Don't treat English DocFX warnings as build failures (for local iteration; full/CI builds stay strict) | | `--sync` | Sync English fallback for missing/outdated translations (for local dev) | +| `--warnings-report PATH` | Write a markdown summary of the DocFX warnings per built language to `PATH` (CI appends it to the job summary); written even when the build fails | ## What the Build Script Does @@ -92,7 +93,7 @@ The deploy step waits for all checks, so a failing check blocks both the product |-----|--------------|------------| | Lint build scripts | `./run scripts check`: ruff, ruff format, mypy `--strict` on the qualified Python sources (see `pyproject.toml`); shellcheck and shfmt on `run` and `build_scripts/run_scripts/*.sh` | any lint, type, or formatting finding | | Doctest C# code blocks | `./run doctest validate` always; `./run doctest` (compile, run, and compare `**Output**` blocks) when a te CLI is available, see below | malformed annotation; with te: compile error, runtime error, or output mismatch | -| Build Documentation | `python build-docs.py --all --skip-gen` on Windows, uploads `_site` | DocFX build failure or English DocFX warnings | +| Build Documentation | `python build-docs.py --all --skip-gen --warnings-report build-warnings.md` on Windows, uploads `_site`; the per-language DocFX warning report goes to the job summary | DocFX build failure or English DocFX warnings | | Check links | `python build_scripts/check_links.py validate stats` against the built `_site` | any broken **internal** link: missing file, missing anchor, or an old root-style path that no longer redirects on the live site | | Deploy to Azure | Azure Static Web Apps upload | only after all of the above pass | @@ -102,6 +103,10 @@ the site is outside our control, and outages, bot-blocking, and link rot should The full report, including the list of external URLs to verify by hand, is attached to the job summary of the "Check links" job. Fix or remove genuinely dead external links as part of normal maintenance. +**Translation build warnings are visible but not blocking.** +Only English DocFX warnings fail the build; the es and zh builds are Crowdin-managed translations and may carry warnings such as broken bookmarks or missing includes. +The "Build Documentation" job summary lists every DocFX warning per language, so translation problems can be spotted and fed back without blocking a deploy. + **Executing the doctests needs the te CLI.** The CLI download is gated behind sign-in, so the workflow cannot fetch it from a public URL. Set the repository secret `TE_CLI_DOWNLOAD_URL` to a direct download URL for `te-linux-x64.tar.gz` (for example a private blob with a SAS token); diff --git a/build-docs.py b/build-docs.py index 91920d985..a94b22e38 100644 --- a/build-docs.py +++ b/build-docs.py @@ -22,6 +22,9 @@ (~30-40% faster). LOCAL markdown iteration ONLY, with --serve/--lang; never for testing, CI/CD, or release builds. --permissive Don't fail the English build on DocFX warnings (local iteration) + --warnings-report PATH + Write a markdown summary of the DocFX warnings per built language + to PATH (for CI job summaries); written even when the build fails """ import argparse @@ -35,21 +38,43 @@ _DOCFX_WARNING_RE = re.compile(r': warning ', re.IGNORECASE) +_ANSI_RE = re.compile(r'\x1b\[[0-9;]*m') +# Language -> DocFX warning diagnostics seen while building it. Filled by +# build_language(); rendered by write_warnings_report() for --warnings-report. +BUILD_WARNINGS: dict[str, list[str]] = {} -def run_command(cmd: list[str], description: str, check: bool = True, fail_on_warnings: bool = False) -> int: + +def _clean_warning_line(line: str) -> str: + """Strip colour codes and the absolute working-directory prefix from a DocFX diagnostic.""" + text = _ANSI_RE.sub('', line).rstrip() + cwd = str(Path.cwd()) + return text.replace(cwd + os.sep, '').replace(cwd + '/', '') + + +def run_command( + cmd: list[str], + description: str, + check: bool = True, + fail_on_warnings: bool = False, + warnings: list[str] | None = None, +) -> int: """Run a command and return exit code. If fail_on_warnings=True, streams output line-by-line, counts DocFX warning diagnostics (lines matching ': warning '), and returns exit code 1 if any are found — even when the process itself exits 0. + + If `warnings` is a list, output is streamed the same way and every warning + diagnostic is appended to it (colour codes and the working directory + stripped) for the --warnings-report summary; the exit code is unaffected. """ print(f"\n{'='*60}") print(f" {description}") print(f"{'='*60}") print(f"Running: {' '.join(cmd)}\n") - if fail_on_warnings: + if fail_on_warnings or warnings is not None: warning_count = 0 process = subprocess.Popen( cmd, @@ -64,13 +89,15 @@ def run_command(cmd: list[str], description: str, check: bool = True, fail_on_wa print(line, end='', flush=True) if _DOCFX_WARNING_RE.search(line): warning_count += 1 + if warnings is not None: + warnings.append(_clean_warning_line(line)) process.wait() if check and process.returncode != 0: print(f"Error: Command failed with exit code {process.returncode}") return process.returncode - if warning_count > 0: + if fail_on_warnings and warning_count > 0: print(f"\nError: DocFX produced {warning_count} warning(s). Failing build.") return 1 @@ -234,13 +261,49 @@ def build_language(lang: str, sync: bool = False, skip_api: bool = False, permis # `docfx build` skips API metadata regeneration and reuses the existing # content/api/*.yml (the _apiSource DLLs don't change between content edits), # which is ~30-40% faster; bare `docfx` regenerates metadata then builds. + BUILD_WARNINGS[lang] = [] return run_docfx( [*(["build"] if skip_api else []), config_path], f"Building {lang} documentation", - fail_on_warnings=(lang == "en" and not permissive) + fail_on_warnings=(lang == "en" and not permissive), + warnings=BUILD_WARNINGS[lang], ) +_REPORT_MAX_LINES = 200 # per language; keeps the job summary well under GitHub's 1 MiB cap + + +def write_warnings_report(path: Path) -> None: + """Write a markdown summary of the DocFX warnings collected per built language. + + Meant for CI job summaries (append the file to $GITHUB_STEP_SUMMARY). English + warnings already fail the build, but the localized builds only warn, so this + is where translation warnings become visible without reading the whole log. + """ + lines = ["## DocFX build warnings", ""] + if not BUILD_WARNINGS: + lines += ["No language build ran.", ""] + else: + lines += ["| Language | Warnings |", "|----------|---------:|"] + lines += [f"| {lang} | {len(found)} |" for lang, found in BUILD_WARNINGS.items()] + lines += [ + "", + "English warnings fail the build. Other languages are translations managed outside " + "this repo, so their warnings are listed here but never block a deploy.", + "", + ] + for lang, found in BUILD_WARNINGS.items(): + if not found: + continue + lines += [f"
{lang}: {len(found)} warning(s)", "", "```text"] + lines += found[:_REPORT_MAX_LINES] + if len(found) > _REPORT_MAX_LINES: + lines.append(f"... {len(found) - _REPORT_MAX_LINES} more; see the build log") + lines += ["```", "", "
", ""] + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + print(f"Warnings report written to {path}") + + def copy_languages_manifest() -> int: """Copy languages.json to _site/ root for runtime access.""" manifest_src = Path("metadata/languages.json") @@ -349,9 +412,20 @@ def main() -> int: parser.add_argument("--skip-api", action="store_true", help="LOCAL markdown iteration only (requires --serve/--lang): reuse existing content/api, ~30-40%% faster. NEVER for testing/CI/CD/releases") parser.add_argument("--permissive", action="store_true", help="Don't treat English DocFX warnings as build failures (for local iteration; keep full/CI builds strict)") parser.add_argument("--sync", action="store_true", help="Sync English fallback for missing/outdated translations (for local dev)") - + parser.add_argument("--warnings-report", metavar="PATH", help="Write a markdown summary of DocFX warnings per built language to PATH (for CI job summaries); written even when the build fails") + args = parser.parse_args() - + + try: + return _build(args) + finally: + # Written even after a failed language build, so CI still gets the partial picture. + if args.warnings_report: + write_warnings_report(Path(args.warnings_report)) + + +def _build(args: argparse.Namespace) -> int: + """Run the build described by the parsed command line; returns the exit code.""" # List available languages if args.list: langs = get_available_languages() From 085483c3927e67a228d649f446f3135c24340fa5 Mon Sep 17 00:00:00 2001 From: Nedas Date: Fri, 4 Sep 2026 12:17:39 +0200 Subject: [PATCH 4/4] Fixed unicode error --- build-docs.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/build-docs.py b/build-docs.py index a94b22e38..d83212661 100644 --- a/build-docs.py +++ b/build-docs.py @@ -45,6 +45,21 @@ BUILD_WARNINGS: dict[str, list[str]] = {} +def _use_utf8_stdio() -> None: + """Make stdout/stderr UTF-8 with replacement for unencodable characters. + + DocFX echoes translated headings and file names (e.g. Chinese) in its + diagnostics, which this script re-prints while streaming. When stdout is + redirected on Windows (CI logs, `| tee`), Python defaults to the ANSI code + page and print() raises UnicodeEncodeError on them, aborting the build over + a log line. A real console is already UTF-16/UTF-8 on Windows, so this only + changes the redirected case. + """ + for stream in (sys.stdout, sys.stderr): + if hasattr(stream, "reconfigure"): + stream.reconfigure(encoding="utf-8", errors="replace") + + def _clean_warning_line(line: str) -> str: """Strip colour codes and the absolute working-directory prefix from a DocFX diagnostic.""" text = _ANSI_RE.sub('', line).rstrip() @@ -398,6 +413,7 @@ def fix_xref_in_api() -> int: def main() -> int: + _use_utf8_stdio() parser = argparse.ArgumentParser( description="Build documentation for one or more languages", formatter_class=argparse.RawDescriptionHelpFormatter,