Pospell check

.github/workflows/spellcheck.yml

@@ -0,0 +1,96 @@
+name: spellcheck
+
+on:
+  pull_request:
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      version:
+        description: CPython documentation version
+        required: true
+        type: string
+      translations-ref:
+        description: Translation branch updated by the calling workflow
+        required: true
+        type: string
+    secrets:
+      TX_TOKEN:
+        description: Transifex API token with access to the Polish terminology glossary
+        required: true
+
+permissions:
+  contents: read
+
+jobs:
+  pospell:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    defaults:
+      run:
+        working-directory: cpython/Doc
+    steps:
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: 3
+      - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d  # v10.0.1
+      - name: Check out CPython documentation
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          repository: python/cpython
+          ref: ${{ inputs.version || github.base_ref || github.ref_name }}
+          path: cpython
+          persist-credentials: false
+      - name: Check out translations
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          ref: ${{ inputs.translations-ref }}
+          path: cpython/Doc/locales/pl/LC_MESSAGES
+          persist-credentials: false
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y hunspell hunspell-pl
+          make venv
+          # Pospell needs newer Docutils than the Sphinx used by older branches.
+          uv venv "$RUNNER_TEMP/spellcheck-venv"
+          uv pip install --python "$RUNNER_TEMP/spellcheck-venv/bin/python" sphinx-intl pospell transifex-python six
+      - name: Prepare UTF-8 Polish dictionary
+        # The packaged dictionary uses ISO-8859-2, which cannot represent all prose.
+        run: |
+          mkdir -p "$RUNNER_TEMP/hunspell-utf8"
+          dictionary_encoding=$(sed -n 's/^SET //p' /usr/share/hunspell/pl_PL.aff)
+          iconv -f "$dictionary_encoding" -t UTF-8 /usr/share/hunspell/pl_PL.aff \
+            > "$RUNNER_TEMP/hunspell-utf8/pl_PL.aff"
+          sed -i 's/^SET .*/SET UTF-8/' "$RUNNER_TEMP/hunspell-utf8/pl_PL.aff"
+          iconv -f "$dictionary_encoding" -t UTF-8 /usr/share/hunspell/pl_PL.dic \
+            > "$RUNNER_TEMP/hunspell-utf8/pl_PL.dic"
+      - name: Download Polish terminology glossary
+        working-directory: cpython/Doc/locales/pl/LC_MESSAGES
+        env:
+          TX_TOKEN: ${{ secrets.TX_TOKEN }}
+        run: >
+          "$RUNNER_TEMP/spellcheck-venv/bin/python"
+          manage_translation.py fetch_glossary
+          --output "$RUNNER_TEMP/hunspell-glossary.dic"
+      - name: Generate POT files without literal blocks
+        run: |
+          if make -n gettext >/dev/null 2>&1; then
+            make gettext SPHINXOPTS="-D gettext_additional_targets=index -D gettext_compact=0"
+          else
+            # CPython 3.10 and 3.11 do not have a dedicated gettext target.
+            make build BUILDER=gettext SPHINXOPTS="-D gettext_additional_targets=index -D gettext_compact=0"
+          fi
+      - name: Update temporary PO files
+        # Drop obsolete entries so removed literal blocks are not spellchecked.
+        run: >
+          "$RUNNER_TEMP/spellcheck-venv/bin/sphinx-intl" update -p build/gettext -d locales
+          -l pl
+          --no-obsolete
+      - name: Check Polish spelling
+        env:
+          DICPATH: ${{ runner.temp }}/hunspell-utf8
+        run: >
+          "$RUNNER_TEMP/spellcheck-venv/bin/pospell" --language pl_PL
+          --personal-dict "$RUNNER_TEMP/hunspell-glossary.dic"
+          --glob 'locales/pl/LC_MESSAGES/**/*.po'
+      # PO updates exist only in this disposable checkout; nothing is committed.

.github/workflows/update-and-build.yml

@@ -72,6 +72,19 @@ jobs:
           branch: ${{ matrix.version }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
+  spellcheck:
+    needs: [update]
+    # These are the runs in which update publishes the fetched translations.
+    if: github.event_name == 'schedule' || github.ref_name == github.event.repository.default_branch
+    permissions:
+      contents: read
+    uses: ./.github/workflows/spellcheck.yml
+    with:
+      version: ${{ github.ref_name }}
+      translations-ref: ${{ github.ref_name }}
+    secrets:
+      TX_TOKEN: ${{ secrets.TX_TOKEN }}
+
   build:
     runs-on: ubuntu-latest
     strategy:

manage_translation.py

@@ -12,24 +12,33 @@
 # * recreate_tx_config: recreate configuration for all resources.
 # * warn_about_files_to_delete: lists files that are not available upstream
 # * generate_commit_msg: generates commit message with co-authors
+# * fetch_glossary: download Polish terminology as a Hunspell personal dictionary
 
 from argparse import ArgumentParser
+import csv
+import io
 import os
 from contextlib import chdir
 from dataclasses import dataclass
 from pathlib import Path
+import re
 from subprocess import call, run, CalledProcessError
 import sys
 from tempfile import TemporaryDirectory
+import time
 from typing import Self, Iterable
+import unicodedata
 from warnings import warn
 
 from polib import pofile, POFile
+import requests
 from transifex.api import transifex_api
+from transifex.api.jsonapi.exceptions import JsonApiException
 
 LANGUAGE = 'pl'
 PROJECT_SLUG = 'python-newest'
 VERSION = '3.14'
+GLOSSARY_ID = 'o:python-doc:g:python_global'
 
 
 def fetch():
@@ -45,6 +54,59 @@ def fetch():
         _call(f'msgcat --no-location -o {file} {file}')
 
 
+def _download_glossary(token: str, *, max_wait: float = 300) -> str:
+    transifex_api.setup(auth=token)
+    # The SDK supports new API resource types before adding dedicated classes.
+    download = transifex_api.new(
+        type='glossaries_async_downloads',
+        glossary=transifex_api.new(type='glossaries', id=GLOSSARY_ID),
+        language=transifex_api.Language(id=f'l:{LANGUAGE}'),
+    )
+    download.save()
+    deadline = time.monotonic() + max_wait
+    while not download.redirect:
+        status = download.attributes['status']
+        if status == 'failed':
+            raise ValueError('Transifex glossary export failed.')
+        if status not in {'pending', 'processing'}:
+            raise ValueError(f'Unexpected glossary export status: {status!r}')
+        if time.monotonic() >= deadline:
+            raise ValueError('Transifex glossary export timed out.')
+        time.sleep(3)
+        download.reload()
+    # Download the CSV without sending the API token to the signed storage URL.
+    export = requests.get(download.redirect, timeout=30)
+    export.raise_for_status()
+    return export.content.decode('utf-8-sig')
+
+
+def _glossary_words(content: str) -> list[str]:
+    reader = csv.DictReader(io.StringIO(content))
+    translation_column = f'translation_{LANGUAGE}'
+    if not reader.fieldnames or translation_column not in reader.fieldnames:
+        raise ValueError('Transifex glossary CSV has no Polish translation column.')
+    words = {
+        word
+        for row in reader
+        for word in re.findall(
+            r'[^\W\d_]+', unicodedata.normalize('NFC', row[translation_column] or '')
+        )
+    }
+    if not words:
+        raise ValueError('Transifex glossary has no Polish terms.')
+    return sorted(words)
+
+
+def fetch_glossary(output: Path):
+    """Fetch Polish terminology and write a UTF-8 Hunspell personal dictionary."""
+    token = _get_tx_token().strip()
+    if not token:
+        raise ValueError('TX_TOKEN is required to download the Transifex glossary.')
+    words = _glossary_words(_download_glossary(token))
+    output.write_text(f'{len(words)}\n' + '\n'.join(words) + '\n', encoding='utf-8')
+    print(f'Added {len(words)} Polish glossary words to {output}.')
+
+
 def _call(command: str):
     if (return_code := call(command, shell=True)) != 0:
         exit(return_code)
@@ -223,10 +285,30 @@ def generate_commit_msg():
         'recreate_tx_config',
         'warn_about_files_to_delete',
         'generate_commit_msg',
+        'fetch_glossary',
     )
 
     parser = ArgumentParser()
     parser.add_argument('cmd', choices=RUNNABLE_SCRIPTS)
+    parser.add_argument(
+        '--output', type=Path, help='Hunspell dictionary path for fetch_glossary'
+    )
     options = parser.parse_args()
 
-    eval(options.cmd)()
+    if options.cmd == 'fetch_glossary':
+        if options.output is None:
+            parser.error('--output is required for fetch_glossary')
+        try:
+            fetch_glossary(options.output)
+        except (requests.RequestException, JsonApiException) as error:
+            # Exceptions may contain signed URLs; report the status without those URLs.
+            status = (
+                error.response.status_code
+                if error.response is not None
+                else 'network error'
+            )
+            parser.exit(1, f'Transifex glossary request failed: {status}.\n')
+        except (ValueError, KeyError) as error:
+            parser.exit(1, f'Invalid Transifex glossary export: {error}\n')
+    else:
+        eval(options.cmd)()