Add Transifex glossary terms as hunspell personal dict

Author: m-aciek

.github/workflows/spellcheck.yml

@@ -13,13 +13,18 @@ on:
         description: Translation branch updated by the calling workflow
         required: true
         type: string
+    secrets:
+      TX_TOKEN:
+        description: Transifex API token with access to the Polish terminology glossary
+        required: true
 
 permissions:
   contents: read
 
 jobs:
   pospell:
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     defaults:
       run:
         working-directory: cpython/Doc
@@ -48,7 +53,7 @@ jobs:
           make venv
           # Pospell needs newer Docutils than the Sphinx used by older branches.
           uv venv "$RUNNER_TEMP/spellcheck-venv"
-          uv pip install --python "$RUNNER_TEMP/spellcheck-venv/bin/python" sphinx-intl pospell
+          uv pip install --python "$RUNNER_TEMP/spellcheck-venv/bin/python" sphinx-intl pospell transifex-python six
       - name: Prepare UTF-8 Polish dictionary
         # The packaged dictionary uses ISO-8859-2, which cannot represent all prose.
         run: |
@@ -59,6 +64,14 @@ jobs:
           sed -i 's/^SET .*/SET UTF-8/' "$RUNNER_TEMP/hunspell-utf8/pl_PL.aff"
           iconv -f "$dictionary_encoding" -t UTF-8 /usr/share/hunspell/pl_PL.dic \
             > "$RUNNER_TEMP/hunspell-utf8/pl_PL.dic"
+      - name: Download Polish terminology glossary
+        working-directory: cpython/Doc/locales/pl/LC_MESSAGES
+        env:
+          TX_TOKEN: ${{ secrets.TX_TOKEN }}
+        run: >
+          "$RUNNER_TEMP/spellcheck-venv/bin/python"
+          manage_translation.py fetch_glossary
+          --output "$RUNNER_TEMP/hunspell-glossary.dic"
       - name: Generate POT files without literal blocks
         run: |
           if make -n gettext >/dev/null 2>&1; then
@@ -78,5 +91,6 @@ jobs:
           DICPATH: ${{ runner.temp }}/hunspell-utf8
         run: >
           "$RUNNER_TEMP/spellcheck-venv/bin/pospell" --language pl_PL
+          --personal-dict "$RUNNER_TEMP/hunspell-glossary.dic"
           --glob 'locales/pl/LC_MESSAGES/**/*.po'
       # PO updates exist only in this disposable checkout; nothing is committed.

.github/workflows/update-and-build.yml

@@ -82,6 +82,8 @@ jobs:
     with:
       version: ${{ github.ref_name }}
       translations-ref: ${{ github.ref_name }}
+    secrets:
+      TX_TOKEN: ${{ secrets.TX_TOKEN }}
 
   build:
     runs-on: ubuntu-latest

manage_translation.py

@@ -12,24 +12,33 @@
 # * recreate_tx_config: recreate configuration for all resources.
 # * warn_about_files_to_delete: lists files that are not available upstream
 # * generate_commit_msg: generates commit message with co-authors
+# * fetch_glossary: download Polish terminology as a Hunspell personal dictionary
 
 from argparse import ArgumentParser
+import csv
+import io
 import os
 from contextlib import chdir
 from dataclasses import dataclass
 from pathlib import Path
+import re
 from subprocess import call, run, CalledProcessError
 import sys
 from tempfile import TemporaryDirectory
+import time
 from typing import Self, Iterable
+import unicodedata
 from warnings import warn
 
 from polib import pofile, POFile
+import requests
 from transifex.api import transifex_api
+from transifex.api.jsonapi.exceptions import JsonApiException
 
 LANGUAGE = 'pl'
 PROJECT_SLUG = 'python-newest'
 VERSION = '3.14'
+GLOSSARY_ID = 'o:python-doc:g:python_global'
 
 
 def fetch():
@@ -45,6 +54,59 @@ def fetch():
         _call(f'msgcat --no-location -o {file} {file}')
 
 
+def _download_glossary(token: str, *, max_wait: float = 300) -> str:
+    transifex_api.setup(auth=token)
+    # The SDK supports new API resource types before adding dedicated classes.
+    download = transifex_api.new(
+        type='glossaries_async_downloads',
+        glossary=transifex_api.new(type='glossaries', id=GLOSSARY_ID),
+        language=transifex_api.Language(id=f'l:{LANGUAGE}'),
+    )
+    download.save()
+    deadline = time.monotonic() + max_wait
+    while not download.redirect:
+        status = download.attributes['status']
+        if status == 'failed':
+            raise ValueError('Transifex glossary export failed.')
+        if status not in {'pending', 'processing'}:
+            raise ValueError(f'Unexpected glossary export status: {status!r}')
+        if time.monotonic() >= deadline:
+            raise ValueError('Transifex glossary export timed out.')
+        time.sleep(3)
+        download.reload()
+    # Download the CSV without sending the API token to the signed storage URL.
+    export = requests.get(download.redirect, timeout=30)
+    export.raise_for_status()
+    return export.content.decode('utf-8-sig')
+
+
+def _glossary_words(content: str) -> list[str]:
+    reader = csv.DictReader(io.StringIO(content))
+    translation_column = f'translation_{LANGUAGE}'
+    if not reader.fieldnames or translation_column not in reader.fieldnames:
+        raise ValueError('Transifex glossary CSV has no Polish translation column.')
+    words = {
+        word
+        for row in reader
+        for word in re.findall(
+            r'[^\W\d_]+', unicodedata.normalize('NFC', row[translation_column] or '')
+        )
+    }
+    if not words:
+        raise ValueError('Transifex glossary has no Polish terms.')
+    return sorted(words)
+
+
+def fetch_glossary(output: Path):
+    """Fetch Polish terminology and write a UTF-8 Hunspell personal dictionary."""
+    token = _get_tx_token().strip()
+    if not token:
+        raise ValueError('TX_TOKEN is required to download the Transifex glossary.')
+    words = _glossary_words(_download_glossary(token))
+    output.write_text(f'{len(words)}\n' + '\n'.join(words) + '\n', encoding='utf-8')
+    print(f'Added {len(words)} Polish glossary words to {output}.')
+
+
 def _call(command: str):
     if (return_code := call(command, shell=True)) != 0:
         exit(return_code)
@@ -223,10 +285,30 @@ def generate_commit_msg():
         'recreate_tx_config',
         'warn_about_files_to_delete',
         'generate_commit_msg',
+        'fetch_glossary',
     )
 
     parser = ArgumentParser()
     parser.add_argument('cmd', choices=RUNNABLE_SCRIPTS)
+    parser.add_argument(
+        '--output', type=Path, help='Hunspell dictionary path for fetch_glossary'
+    )
     options = parser.parse_args()
 
-    eval(options.cmd)()
+    if options.cmd == 'fetch_glossary':
+        if options.output is None:
+            parser.error('--output is required for fetch_glossary')
+        try:
+            fetch_glossary(options.output)
+        except (requests.RequestException, JsonApiException) as error:
+            # Exceptions may contain signed URLs; report the status without those URLs.
+            status = (
+                error.response.status_code
+                if error.response is not None
+                else 'network error'
+            )
+            parser.exit(1, f'Transifex glossary request failed: {status}.\n')
+        except (ValueError, KeyError) as error:
+            parser.exit(1, f'Invalid Transifex glossary export: {error}\n')
+    else:
+        eval(options.cmd)()