diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml new file mode 100644 index 000000000..0b63bffae --- /dev/null +++ b/.github/workflows/spellcheck.yml @@ -0,0 +1,96 @@ +name: spellcheck + +on: + pull_request: + workflow_dispatch: + workflow_call: + inputs: + version: + description: CPython documentation version + required: true + type: string + translations-ref: + description: Translation branch updated by the calling workflow + required: true + type: string + secrets: + TX_TOKEN: + description: Transifex API token with access to the Polish terminology glossary + required: true + +permissions: + contents: read + +jobs: + pospell: + runs-on: ubuntu-latest + timeout-minutes: 15 + defaults: + run: + working-directory: cpython/Doc + steps: + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: 3 + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - name: Check out CPython documentation + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: python/cpython + ref: ${{ inputs.version || github.base_ref || github.ref_name }} + path: cpython + persist-credentials: false + - name: Check out translations + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.translations-ref }} + path: cpython/Doc/locales/pl/LC_MESSAGES + persist-credentials: false + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y hunspell hunspell-pl + make venv + # Pospell needs newer Docutils than the Sphinx used by older branches. + uv venv "$RUNNER_TEMP/spellcheck-venv" + uv pip install --python "$RUNNER_TEMP/spellcheck-venv/bin/python" sphinx-intl pospell transifex-python six + - name: Prepare UTF-8 Polish dictionary + # The packaged dictionary uses ISO-8859-2, which cannot represent all prose. + run: | + mkdir -p "$RUNNER_TEMP/hunspell-utf8" + dictionary_encoding=$(sed -n 's/^SET //p' /usr/share/hunspell/pl_PL.aff) + iconv -f "$dictionary_encoding" -t UTF-8 /usr/share/hunspell/pl_PL.aff \ + > "$RUNNER_TEMP/hunspell-utf8/pl_PL.aff" + sed -i 's/^SET .*/SET UTF-8/' "$RUNNER_TEMP/hunspell-utf8/pl_PL.aff" + iconv -f "$dictionary_encoding" -t UTF-8 /usr/share/hunspell/pl_PL.dic \ + > "$RUNNER_TEMP/hunspell-utf8/pl_PL.dic" + - name: Download Polish terminology glossary + working-directory: cpython/Doc/locales/pl/LC_MESSAGES + env: + TX_TOKEN: ${{ secrets.TX_TOKEN }} + run: > + "$RUNNER_TEMP/spellcheck-venv/bin/python" + manage_translation.py fetch_glossary + --output "$RUNNER_TEMP/hunspell-glossary.dic" + - name: Generate POT files without literal blocks + run: | + if make -n gettext >/dev/null 2>&1; then + make gettext SPHINXOPTS="-D gettext_additional_targets=index -D gettext_compact=0" + else + # CPython 3.10 and 3.11 do not have a dedicated gettext target. + make build BUILDER=gettext SPHINXOPTS="-D gettext_additional_targets=index -D gettext_compact=0" + fi + - name: Update temporary PO files + # Drop obsolete entries so removed literal blocks are not spellchecked. + run: > + "$RUNNER_TEMP/spellcheck-venv/bin/sphinx-intl" update -p build/gettext -d locales + -l pl + --no-obsolete + - name: Check Polish spelling + env: + DICPATH: ${{ runner.temp }}/hunspell-utf8 + run: > + "$RUNNER_TEMP/spellcheck-venv/bin/pospell" --language pl_PL + --personal-dict "$RUNNER_TEMP/hunspell-glossary.dic" + --glob 'locales/pl/LC_MESSAGES/**/*.po' + # PO updates exist only in this disposable checkout; nothing is committed. diff --git a/.github/workflows/update-and-build.yml b/.github/workflows/update-and-build.yml index aa8e6695b..35db30528 100644 --- a/.github/workflows/update-and-build.yml +++ b/.github/workflows/update-and-build.yml @@ -72,6 +72,19 @@ jobs: branch: ${{ matrix.version }} github_token: ${{ secrets.GITHUB_TOKEN }} + spellcheck: + needs: [update] + # These are the runs in which update publishes the fetched translations. + if: github.event_name == 'schedule' || github.ref_name == github.event.repository.default_branch + permissions: + contents: read + uses: ./.github/workflows/spellcheck.yml + with: + version: ${{ github.ref_name }} + translations-ref: ${{ github.ref_name }} + secrets: + TX_TOKEN: ${{ secrets.TX_TOKEN }} + build: runs-on: ubuntu-latest strategy: diff --git a/manage_translation.py b/manage_translation.py index 0266629e4..2a209bbff 100755 --- a/manage_translation.py +++ b/manage_translation.py @@ -12,24 +12,33 @@ # * recreate_tx_config: recreate configuration for all resources. # * warn_about_files_to_delete: lists files that are not available upstream # * generate_commit_msg: generates commit message with co-authors +# * fetch_glossary: download Polish terminology as a Hunspell personal dictionary from argparse import ArgumentParser +import csv +import io import os from contextlib import chdir from dataclasses import dataclass from pathlib import Path +import re from subprocess import call, run, CalledProcessError import sys from tempfile import TemporaryDirectory +import time from typing import Self, Iterable +import unicodedata from warnings import warn from polib import pofile, POFile +import requests from transifex.api import transifex_api +from transifex.api.jsonapi.exceptions import JsonApiException LANGUAGE = 'pl' PROJECT_SLUG = 'python-newest' VERSION = '3.14' +GLOSSARY_ID = 'o:python-doc:g:python_global' def fetch(): @@ -45,6 +54,59 @@ def fetch(): _call(f'msgcat --no-location -o {file} {file}') +def _download_glossary(token: str, *, max_wait: float = 300) -> str: + transifex_api.setup(auth=token) + # The SDK supports new API resource types before adding dedicated classes. + download = transifex_api.new( + type='glossaries_async_downloads', + glossary=transifex_api.new(type='glossaries', id=GLOSSARY_ID), + language=transifex_api.Language(id=f'l:{LANGUAGE}'), + ) + download.save() + deadline = time.monotonic() + max_wait + while not download.redirect: + status = download.attributes['status'] + if status == 'failed': + raise ValueError('Transifex glossary export failed.') + if status not in {'pending', 'processing'}: + raise ValueError(f'Unexpected glossary export status: {status!r}') + if time.monotonic() >= deadline: + raise ValueError('Transifex glossary export timed out.') + time.sleep(3) + download.reload() + # Download the CSV without sending the API token to the signed storage URL. + export = requests.get(download.redirect, timeout=30) + export.raise_for_status() + return export.content.decode('utf-8-sig') + + +def _glossary_words(content: str) -> list[str]: + reader = csv.DictReader(io.StringIO(content)) + translation_column = f'translation_{LANGUAGE}' + if not reader.fieldnames or translation_column not in reader.fieldnames: + raise ValueError('Transifex glossary CSV has no Polish translation column.') + words = { + word + for row in reader + for word in re.findall( + r'[^\W\d_]+', unicodedata.normalize('NFC', row[translation_column] or '') + ) + } + if not words: + raise ValueError('Transifex glossary has no Polish terms.') + return sorted(words) + + +def fetch_glossary(output: Path): + """Fetch Polish terminology and write a UTF-8 Hunspell personal dictionary.""" + token = _get_tx_token().strip() + if not token: + raise ValueError('TX_TOKEN is required to download the Transifex glossary.') + words = _glossary_words(_download_glossary(token)) + output.write_text(f'{len(words)}\n' + '\n'.join(words) + '\n', encoding='utf-8') + print(f'Added {len(words)} Polish glossary words to {output}.') + + def _call(command: str): if (return_code := call(command, shell=True)) != 0: exit(return_code) @@ -223,10 +285,30 @@ def generate_commit_msg(): 'recreate_tx_config', 'warn_about_files_to_delete', 'generate_commit_msg', + 'fetch_glossary', ) parser = ArgumentParser() parser.add_argument('cmd', choices=RUNNABLE_SCRIPTS) + parser.add_argument( + '--output', type=Path, help='Hunspell dictionary path for fetch_glossary' + ) options = parser.parse_args() - eval(options.cmd)() + if options.cmd == 'fetch_glossary': + if options.output is None: + parser.error('--output is required for fetch_glossary') + try: + fetch_glossary(options.output) + except (requests.RequestException, JsonApiException) as error: + # Exceptions may contain signed URLs; report the status without those URLs. + status = ( + error.response.status_code + if error.response is not None + else 'network error' + ) + parser.exit(1, f'Transifex glossary request failed: {status}.\n') + except (ValueError, KeyError) as error: + parser.exit(1, f'Invalid Transifex glossary export: {error}\n') + else: + eval(options.cmd)()