From e389ed5a4e0c556e7ce6fcba48daafe4e21764de Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Thu, 1 Oct 2026 09:50:00 +0200 Subject: [PATCH 1/5] Run pospell/hunspell skipping literal blocks --- .github/workflows/spellcheck.yml | 70 ++++++++++++++++++++++++++ .github/workflows/update-and-build.yml | 11 ++++ 2 files changed, 81 insertions(+) create mode 100644 .github/workflows/spellcheck.yml diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml new file mode 100644 index 000000000..f678b5b19 --- /dev/null +++ b/.github/workflows/spellcheck.yml @@ -0,0 +1,70 @@ +name: spellcheck + +on: + pull_request: + workflow_dispatch: + workflow_call: + inputs: + version: + description: CPython documentation version + required: true + type: string + translations-ref: + description: Translation branch updated by the calling workflow + required: true + type: string + +permissions: + contents: read + +jobs: + pospell: + runs-on: ubuntu-latest + defaults: + run: + working-directory: cpython/Doc + steps: + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - name: Check out CPython documentation + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: python/cpython + ref: ${{ inputs.version || github.base_ref || github.ref_name }} + path: cpython + persist-credentials: false + - name: Check out translations + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.translations-ref }} + path: cpython/Doc/locales/pl/LC_MESSAGES + persist-credentials: false + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y hunspell hunspell-pl + make venv + # Pospell needs newer Docutils than the Sphinx used by older branches. + uv venv "$RUNNER_TEMP/spellcheck-venv" + uv pip install --python "$RUNNER_TEMP/spellcheck-venv/bin/python" sphinx-intl pospell + - name: Generate POT files without literal blocks + run: | + if make -n gettext >/dev/null 2>&1; then + make gettext SPHINXOPTS="-D gettext_additional_targets=index -D gettext_compact=False" + else + # CPython 3.10 and 3.11 do not have a dedicated gettext target. + make build BUILDER=gettext SPHINXOPTS="-D gettext_additional_targets=index -D gettext_compact=False" + fi + - name: Update temporary PO files + # Drop obsolete entries so removed literal blocks are not spellchecked. + run: > + "$RUNNER_TEMP/spellcheck-venv/bin/sphinx-intl" update -p build/gettext -d locales + -l pl + --no-obsolete + - name: Check Polish spelling + run: > + "$RUNNER_TEMP/spellcheck-venv/bin/pospell" --language pl + --glob 'locales/pl/LC_MESSAGES/**/*.po' + # PO updates exist only in this disposable checkout; nothing is committed. diff --git a/.github/workflows/update-and-build.yml b/.github/workflows/update-and-build.yml index aa8e6695b..69de933fd 100644 --- a/.github/workflows/update-and-build.yml +++ b/.github/workflows/update-and-build.yml @@ -72,6 +72,17 @@ jobs: branch: ${{ matrix.version }} github_token: ${{ secrets.GITHUB_TOKEN }} + spellcheck: + needs: [update] + # These are the runs in which update publishes the fetched translations. + if: github.event_name == 'schedule' || github.ref_name == github.event.repository.default_branch + permissions: + contents: read + uses: ./.github/workflows/spellcheck.yml + with: + version: ${{ github.ref_name }} + translations-ref: ${{ github.ref_name }} + build: runs-on: ubuntu-latest strategy: From 9832a9b7d0a7f72872117eee0518ffd2bce840ba Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Thu, 1 Oct 2026 22:10:01 +0200 Subject: [PATCH 2/5] Unpin Python version for builds --- .github/workflows/spellcheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index f678b5b19..f9a12282d 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -26,7 +26,7 @@ jobs: steps: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version: '3.12' + python-version: 3 - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - name: Check out CPython documentation uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 From 81890bfe178914a5a028849095f30f36250f4203 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Thu, 1 Oct 2026 22:34:14 +0200 Subject: [PATCH 3/5] Fix gettext_compact=0, use pl_PL for pospell --- .github/workflows/spellcheck.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index f9a12282d..98539b1c3 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -52,10 +52,10 @@ jobs: - name: Generate POT files without literal blocks run: | if make -n gettext >/dev/null 2>&1; then - make gettext SPHINXOPTS="-D gettext_additional_targets=index -D gettext_compact=False" + make gettext SPHINXOPTS="-D gettext_additional_targets=index -D gettext_compact=0" else # CPython 3.10 and 3.11 do not have a dedicated gettext target. - make build BUILDER=gettext SPHINXOPTS="-D gettext_additional_targets=index -D gettext_compact=False" + make build BUILDER=gettext SPHINXOPTS="-D gettext_additional_targets=index -D gettext_compact=0" fi - name: Update temporary PO files # Drop obsolete entries so removed literal blocks are not spellchecked. @@ -65,6 +65,6 @@ jobs: --no-obsolete - name: Check Polish spelling run: > - "$RUNNER_TEMP/spellcheck-venv/bin/pospell" --language pl + "$RUNNER_TEMP/spellcheck-venv/bin/pospell" --language pl_PL --glob 'locales/pl/LC_MESSAGES/**/*.po' # PO updates exist only in this disposable checkout; nothing is committed. From 2f1e1106722fe996935a804ba7b8a2970ac6448b Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Thu, 1 Oct 2026 22:45:12 +0200 Subject: [PATCH 4/5] Prepare UTF-8 Polish dictionary By default, ISO-8859-2 is used, for it the check fails on some documentation characters. --- .github/workflows/spellcheck.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 98539b1c3..d232cc97d 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -49,6 +49,16 @@ jobs: # Pospell needs newer Docutils than the Sphinx used by older branches. uv venv "$RUNNER_TEMP/spellcheck-venv" uv pip install --python "$RUNNER_TEMP/spellcheck-venv/bin/python" sphinx-intl pospell + - name: Prepare UTF-8 Polish dictionary + # The packaged dictionary uses ISO-8859-2, which cannot represent all prose. + run: | + mkdir -p "$RUNNER_TEMP/hunspell-utf8" + dictionary_encoding=$(sed -n 's/^SET //p' /usr/share/hunspell/pl_PL.aff) + iconv -f "$dictionary_encoding" -t UTF-8 /usr/share/hunspell/pl_PL.aff \ + > "$RUNNER_TEMP/hunspell-utf8/pl_PL.aff" + sed -i 's/^SET .*/SET UTF-8/' "$RUNNER_TEMP/hunspell-utf8/pl_PL.aff" + iconv -f "$dictionary_encoding" -t UTF-8 /usr/share/hunspell/pl_PL.dic \ + > "$RUNNER_TEMP/hunspell-utf8/pl_PL.dic" - name: Generate POT files without literal blocks run: | if make -n gettext >/dev/null 2>&1; then @@ -64,6 +74,8 @@ jobs: -l pl --no-obsolete - name: Check Polish spelling + env: + DICPATH: ${{ runner.temp }}/hunspell-utf8 run: > "$RUNNER_TEMP/spellcheck-venv/bin/pospell" --language pl_PL --glob 'locales/pl/LC_MESSAGES/**/*.po' From 929849d439164a3c433c150cdf3deb05d1d5d02a Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Fri, 2 Oct 2026 00:20:39 +0200 Subject: [PATCH 5/5] Add Transifex glossary terms as hunspell personal dict --- .github/workflows/spellcheck.yml | 16 ++++- .github/workflows/update-and-build.yml | 2 + manage_translation.py | 84 +++++++++++++++++++++++++- 3 files changed, 100 insertions(+), 2 deletions(-) diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index d232cc97d..0b63bffae 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -13,6 +13,10 @@ on: description: Translation branch updated by the calling workflow required: true type: string + secrets: + TX_TOKEN: + description: Transifex API token with access to the Polish terminology glossary + required: true permissions: contents: read @@ -20,6 +24,7 @@ permissions: jobs: pospell: runs-on: ubuntu-latest + timeout-minutes: 15 defaults: run: working-directory: cpython/Doc @@ -48,7 +53,7 @@ jobs: make venv # Pospell needs newer Docutils than the Sphinx used by older branches. uv venv "$RUNNER_TEMP/spellcheck-venv" - uv pip install --python "$RUNNER_TEMP/spellcheck-venv/bin/python" sphinx-intl pospell + uv pip install --python "$RUNNER_TEMP/spellcheck-venv/bin/python" sphinx-intl pospell transifex-python six - name: Prepare UTF-8 Polish dictionary # The packaged dictionary uses ISO-8859-2, which cannot represent all prose. run: | @@ -59,6 +64,14 @@ jobs: sed -i 's/^SET .*/SET UTF-8/' "$RUNNER_TEMP/hunspell-utf8/pl_PL.aff" iconv -f "$dictionary_encoding" -t UTF-8 /usr/share/hunspell/pl_PL.dic \ > "$RUNNER_TEMP/hunspell-utf8/pl_PL.dic" + - name: Download Polish terminology glossary + working-directory: cpython/Doc/locales/pl/LC_MESSAGES + env: + TX_TOKEN: ${{ secrets.TX_TOKEN }} + run: > + "$RUNNER_TEMP/spellcheck-venv/bin/python" + manage_translation.py fetch_glossary + --output "$RUNNER_TEMP/hunspell-glossary.dic" - name: Generate POT files without literal blocks run: | if make -n gettext >/dev/null 2>&1; then @@ -78,5 +91,6 @@ jobs: DICPATH: ${{ runner.temp }}/hunspell-utf8 run: > "$RUNNER_TEMP/spellcheck-venv/bin/pospell" --language pl_PL + --personal-dict "$RUNNER_TEMP/hunspell-glossary.dic" --glob 'locales/pl/LC_MESSAGES/**/*.po' # PO updates exist only in this disposable checkout; nothing is committed. diff --git a/.github/workflows/update-and-build.yml b/.github/workflows/update-and-build.yml index 69de933fd..35db30528 100644 --- a/.github/workflows/update-and-build.yml +++ b/.github/workflows/update-and-build.yml @@ -82,6 +82,8 @@ jobs: with: version: ${{ github.ref_name }} translations-ref: ${{ github.ref_name }} + secrets: + TX_TOKEN: ${{ secrets.TX_TOKEN }} build: runs-on: ubuntu-latest diff --git a/manage_translation.py b/manage_translation.py index 0266629e4..2a209bbff 100755 --- a/manage_translation.py +++ b/manage_translation.py @@ -12,24 +12,33 @@ # * recreate_tx_config: recreate configuration for all resources. # * warn_about_files_to_delete: lists files that are not available upstream # * generate_commit_msg: generates commit message with co-authors +# * fetch_glossary: download Polish terminology as a Hunspell personal dictionary from argparse import ArgumentParser +import csv +import io import os from contextlib import chdir from dataclasses import dataclass from pathlib import Path +import re from subprocess import call, run, CalledProcessError import sys from tempfile import TemporaryDirectory +import time from typing import Self, Iterable +import unicodedata from warnings import warn from polib import pofile, POFile +import requests from transifex.api import transifex_api +from transifex.api.jsonapi.exceptions import JsonApiException LANGUAGE = 'pl' PROJECT_SLUG = 'python-newest' VERSION = '3.14' +GLOSSARY_ID = 'o:python-doc:g:python_global' def fetch(): @@ -45,6 +54,59 @@ def fetch(): _call(f'msgcat --no-location -o {file} {file}') +def _download_glossary(token: str, *, max_wait: float = 300) -> str: + transifex_api.setup(auth=token) + # The SDK supports new API resource types before adding dedicated classes. + download = transifex_api.new( + type='glossaries_async_downloads', + glossary=transifex_api.new(type='glossaries', id=GLOSSARY_ID), + language=transifex_api.Language(id=f'l:{LANGUAGE}'), + ) + download.save() + deadline = time.monotonic() + max_wait + while not download.redirect: + status = download.attributes['status'] + if status == 'failed': + raise ValueError('Transifex glossary export failed.') + if status not in {'pending', 'processing'}: + raise ValueError(f'Unexpected glossary export status: {status!r}') + if time.monotonic() >= deadline: + raise ValueError('Transifex glossary export timed out.') + time.sleep(3) + download.reload() + # Download the CSV without sending the API token to the signed storage URL. + export = requests.get(download.redirect, timeout=30) + export.raise_for_status() + return export.content.decode('utf-8-sig') + + +def _glossary_words(content: str) -> list[str]: + reader = csv.DictReader(io.StringIO(content)) + translation_column = f'translation_{LANGUAGE}' + if not reader.fieldnames or translation_column not in reader.fieldnames: + raise ValueError('Transifex glossary CSV has no Polish translation column.') + words = { + word + for row in reader + for word in re.findall( + r'[^\W\d_]+', unicodedata.normalize('NFC', row[translation_column] or '') + ) + } + if not words: + raise ValueError('Transifex glossary has no Polish terms.') + return sorted(words) + + +def fetch_glossary(output: Path): + """Fetch Polish terminology and write a UTF-8 Hunspell personal dictionary.""" + token = _get_tx_token().strip() + if not token: + raise ValueError('TX_TOKEN is required to download the Transifex glossary.') + words = _glossary_words(_download_glossary(token)) + output.write_text(f'{len(words)}\n' + '\n'.join(words) + '\n', encoding='utf-8') + print(f'Added {len(words)} Polish glossary words to {output}.') + + def _call(command: str): if (return_code := call(command, shell=True)) != 0: exit(return_code) @@ -223,10 +285,30 @@ def generate_commit_msg(): 'recreate_tx_config', 'warn_about_files_to_delete', 'generate_commit_msg', + 'fetch_glossary', ) parser = ArgumentParser() parser.add_argument('cmd', choices=RUNNABLE_SCRIPTS) + parser.add_argument( + '--output', type=Path, help='Hunspell dictionary path for fetch_glossary' + ) options = parser.parse_args() - eval(options.cmd)() + if options.cmd == 'fetch_glossary': + if options.output is None: + parser.error('--output is required for fetch_glossary') + try: + fetch_glossary(options.output) + except (requests.RequestException, JsonApiException) as error: + # Exceptions may contain signed URLs; report the status without those URLs. + status = ( + error.response.status_code + if error.response is not None + else 'network error' + ) + parser.exit(1, f'Transifex glossary request failed: {status}.\n') + except (ValueError, KeyError) as error: + parser.exit(1, f'Invalid Transifex glossary export: {error}\n') + else: + eval(options.cmd)()