Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
276 changes: 276 additions & 0 deletions .github/workflows/python.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,276 @@
name: python

on:
push:
release:
types:
- published

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true

env:
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_MAXSIZE: 1G
CCACHE_KEY_SUFFIX: r1
CONAN_HOME: ${{ github.workspace }}/.conan2
CONAN_KEY_SUFFIX: r1

jobs:
format:
runs-on: ubuntu-24.04
steps:
- name: checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

- name: setup python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
with:
python-version: 3.14
- name: install black
run: pip install black

- name: check python formatting
run: black --check --diff python

build-test:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
include:
- { os: ubuntu-24.04, build_profile: ubuntu-24.04-clang-18, host_profile: ubuntu-24.04-clang-18 }
- { os: macos-26, build_profile: macos-26-armv8-clang-14, host_profile: macos-26-armv8-clang-14 }
steps:
- name: checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
- name: checkout conan-odr-index
run: git submodule update --init --depth 1 conan-odr-index

- name: ubuntu install ccache
if: runner.os == 'Linux'
run: |
sudo apt install ccache
ccache -V
- name: macos install ccache
if: runner.os == 'macOS'
run: |
brew install ccache
ccache -V

- name: setup python 3.14
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
with:
python-version: 3.14
- name: install python dependencies
run: pip install conan pytest

- name: cache conan
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
with:
path: ${{ env.CONAN_HOME }}
key: conan-python-${{ matrix.host_profile }}-${{ env.CONAN_KEY_SUFFIX }}
restore-keys: |
conan-python-${{ matrix.host_profile }}-
conan-${{ matrix.host_profile }}-

- name: export conan-odr-index
run: python conan-odr-index/scripts/conan_export_all_packages.py --selection-config conan-odr-index/defaults.yaml
- name: conan config
run: conan config install .github/config/conan

- name: cache ccache
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
with:
path: ${{ env.CCACHE_DIR }}
key: ccache-python-${{ matrix.host_profile }}-${{ env.CCACHE_KEY_SUFFIX }}
restore-keys: |
ccache-python-${{ matrix.host_profile }}-
ccache-${{ matrix.host_profile }}-

- name: conan install
run: >
conan install .
-o '&:with_python=True'
--profile:host '${{ matrix.host_profile }}'
--profile:build '${{ matrix.build_profile }}'
--build missing

- name: cmake
run: >
cmake -B build -S .
-DCMAKE_TOOLCHAIN_FILE="conan_toolchain.cmake"
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_CXX_FLAGS="-Werror"
-DODR_PYTHON=ON
-DODR_CLI=OFF
-DODR_TEST=OFF

- name: build
run: cmake --build build --target pyodr_core --config Release

- name: pytest
run: PYTHONPATH=build/python python -m pytest python/tests -v

wheels:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
include:
- { os: ubuntu-24.04, build_profile: ubuntu-24.04-clang-18, host_profile: ubuntu-24.04-clang-18 }
- { os: macos-26, build_profile: macos-26-armv8-clang-14, host_profile: macos-26-armv8-clang-14 }
steps:
- name: checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
# setuptools-scm derives the package version from git tags.
fetch-depth: 0
- name: checkout conan-odr-index
run: git submodule update --init --depth 1 conan-odr-index

# The conan profiles use ccache as compiler launcher, so it must exist
# even for `--build missing` source builds.
- name: ubuntu install ccache
if: runner.os == 'Linux'
run: |
sudo apt install ccache
ccache -V
- name: macos install ccache
if: runner.os == 'macOS'
run: |
brew install ccache
ccache -V

- name: setup python 3.14
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
with:
python-version: 3.14
- name: install python dependencies
run: pip install conan build pytest

- name: cache conan
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
with:
path: ${{ env.CONAN_HOME }}
key: conan-wheel-${{ matrix.host_profile }}-${{ env.CONAN_KEY_SUFFIX }}
restore-keys: |
conan-wheel-${{ matrix.host_profile }}-

- name: export conan-odr-index
run: python conan-odr-index/scripts/conan_export_all_packages.py --selection-config conan-odr-index/defaults.yaml
- name: conan config
run: conan config install .github/config/conan

- name: cache ccache
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
with:
path: ${{ env.CCACHE_DIR }}
key: ccache-wheel-${{ matrix.host_profile }}-${{ env.CCACHE_KEY_SUFFIX }}
restore-keys: |
ccache-wheel-${{ matrix.host_profile }}-
ccache-${{ matrix.host_profile }}-

# Match the dependency set that pyproject.toml enables (no pdf2htmlEX,
# wvWare, libmagic — their runtime data cannot ship inside the wheel).
- name: conan install
run: >
conan install .
-o '&:with_python=True'
-o '&:with_pdf2htmlEX=False'
-o '&:with_wvWare=False'
-o '&:with_libmagic=False'
--profile:host '${{ matrix.host_profile }}'
--profile:build '${{ matrix.build_profile }}'
--build missing

- name: build wheel
run: python -m build --wheel
env:
CMAKE_ARGS: -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/conan_toolchain.cmake

# PyPI rejects plain `linux_*` platform tags; auditwheel grafts the wheel
# onto the matching manylinux tag (the conan deps are linked statically,
# so there should be no external libraries to vendor).
- name: audit wheel
if: runner.os == 'Linux'
run: |
sudo apt install patchelf
pipx run auditwheel repair --wheel-dir dist-audited dist/*.whl
rm dist/*.whl
mv dist-audited/*.whl dist/

- name: test wheel
run: |
pip install dist/*.whl
python -m pytest python/tests -v

- name: upload wheels
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: wheel-${{ matrix.host_profile }}
path: dist/*.whl
if-no-files-found: error

sdist:
runs-on: ubuntu-24.04
steps:
- name: checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
# setuptools-scm derives the package version from git tags.
fetch-depth: 0

- name: build sdist
run: pipx run build --sdist
- name: check metadata
run: pipx run twine check dist/*

- name: upload sdist
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: sdist
path: dist/*.tar.gz
if-no-files-found: error

publish:
needs: [build-test, wheels, sdist]
runs-on: ubuntu-24.04
if: github.event_name == 'release' && github.event.action == 'published'
permissions:
contents: write
steps:
- name: download artifacts
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
path: dist
merge-multiple: true

- name: upload release assets
env:
GH_TOKEN: ${{ github.token }}
run: gh release upload '${{ github.event.release.tag_name }}' dist/* --repo '${{ github.repository }}'

# Uses PyPI trusted publishing (OIDC, no token): the `pyodr` project on PyPI
# must list this repo + workflow + the `pypi` environment as a publisher.
# Note pip cannot build the published sdist on its own (the build needs a
# conan-generated toolchain plus the conan-odr-index recipes, see
# python/README.md); it is published for completeness.
pypi:
needs: [build-test, wheels, sdist]
runs-on: ubuntu-24.04
if: github.event_name == 'release' && github.event.action == 'published'
environment: pypi
permissions:
id-token: write
steps:
- name: download artifacts
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
path: dist
merge-multiple: true

- name: publish to PyPI
uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # v1.14.1
6 changes: 6 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -67,3 +67,9 @@ tools/pdf/afm/
__pycache__/
*.py[cod]
*$py.class

# Packaging / testing
dist/
wheelhouse/
*.egg-info/
.pytest_cache/
5 changes: 3 additions & 2 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ bytes ─▶ magic/open_strategy ─▶ DecodedFile ─▶ Document ─▶ Eleme
| `src/odr/internal/pdf/`, `pdf_poppler/` | PDF (own parser + poppler/pdf2htmlEX path). |
| `src/odr/internal/{csv,json,text,svm}/` | Smaller formats. |
| `cli/src/` | CLI tools: `translate`, `back_translate`, `meta`, `server`. |
| `python/` | Python bindings (`pyodr`, pybind11); see [`python/AGENTS.md`](python/AGENTS.md). |
| `tools/pdf/` | Dev tooling (not built): PDF encoding-data generators, see `tools/pdf/README.md`. |
| `test/src/` | GoogleTest suites; data in `test/data` (git submodules). |
| `offline/documentation/MS-*/` | Vendored Microsoft spec text (see [Specs](#specs)). |
Expand All @@ -85,8 +86,8 @@ cmake --build cmake-build-relwithdebinfo --target translate # CLI: file → HTM
- **Run the test binary from the build dir** so output stays out of the repo tree.
- **For debugging, prefer the `translate` CLI** on a single file over the suite.
- CMake options (`CMakeLists.txt`): `ODR_TEST`, `ODR_CLI`, `ODR_WITH_PDF2HTMLEX`,
`ODR_WITH_WVWARE`, `ODR_WITH_LIBMAGIC`, `ODR_CLANG_TIDY`. A new `.cpp` must be
added to `ODR_SOURCE_FILES`.
`ODR_WITH_WVWARE`, `ODR_WITH_LIBMAGIC`, `ODR_PYTHON`, `ODR_CLANG_TIDY`. A new
`.cpp` must be added to `ODR_SOURCE_FILES`.
- **Test data lives in git submodules** under `test/data/`.

## Conventions
Expand Down
11 changes: 10 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ option(ODR_WITH_PDF2HTMLEX "Build with pdf2htmlEX" "${WITH_PDF2HTMLEX}")
option(ODR_WITH_WVWARE "Build with wvWare" "${WITH_WVWARE}")
option(ODR_WITH_LIBMAGIC "Build with libmagic" "${WITH_LIBMAGIC}")
option(ODR_BUNDLE_ASSETS "Bundle assets during build and install" OFF)
option(ODR_PYTHON "Build Python bindings" OFF)

include(GNUInstallDirs)

Expand Down Expand Up @@ -405,6 +406,10 @@ if (ODR_CLI)
add_subdirectory("cli")
endif ()

if (ODR_PYTHON)
add_subdirectory("python")
endif ()

if (ODR_TEST)
add_subdirectory("test")
endif ()
Expand All @@ -423,8 +428,12 @@ install(
FILES_MATCHING PATTERN "*.hpp"
${ODR_HEADER_EXCLUDE}
)
set(ODR_INSTALL_TARGETS odr)
if (ODR_CLI)
list(APPEND ODR_INSTALL_TARGETS meta translate back_translate)
endif ()
install(
TARGETS odr meta translate back_translate
TARGETS ${ODR_INSTALL_TARGETS}
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
BUNDLE DESTINATION "${CMAKE_INSTALL_BINDIR}"
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
Expand Down
Loading
Loading