diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 117ae0ab..49bde477 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -6,6 +6,18 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +# Cache keys, in short: `actions/cache` never overwrites an existing key, so a +# key that stays the same forever is written exactly once and then frozen — +# every later run restores that first snapshot and rebuilds whatever is missing +# from source. Both caches therefore vary their key: +# * conan — keyed on what determines its content (recipes + profile), so it is +# rewritten precisely when the dependencies change. +# * ccache — keyed per run and restored via prefix, so it keeps growing. Only +# the default branch writes it; branch-scoped entries are invisible to other +# branches and would just churn the 10 GB repo quota. +# `CACHE_FLAVOR` separates jobs that resolve a *different* dependency set for the +# same profile — without it they overwrite each other and both keep rebuilding. +# The `*_KEY_SUFFIX` variables stay as a manual "throw it all away" knob. env: CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 1G @@ -14,16 +26,22 @@ env: CONAN_KEY_SUFFIX: r1 jobs: + # Builds the core once per profile and, where `bindings` is set, the JNI and + # Python bindings out of the same configure. They used to live in their own + # workflows, which meant compiling the core and resolving its dependencies + # three times over for the same two profiles. build: runs-on: ${{ matrix.os }} + env: + CACHE_FLAVOR: main${{ matrix.bindings && '-bindings' || '' }} strategy: fail-fast: false matrix: include: - - { os: ubuntu-24.04, build_profile: ubuntu-24.04-clang-18, host_profile: ubuntu-24.04-clang-18 } + - { os: ubuntu-24.04, build_profile: ubuntu-24.04-clang-18, host_profile: ubuntu-24.04-clang-18, bindings: true } - { os: ubuntu-24.04, build_profile: ubuntu-24.04-gcc-14, host_profile: ubuntu-24.04-gcc-14 } - { os: macos-15, build_profile: macos-15-armv8-clang-14, host_profile: macos-15-armv8-clang-14 } - - { os: macos-26, build_profile: macos-26-armv8-clang-14, host_profile: macos-26-armv8-clang-14 } + - { os: macos-26, build_profile: macos-26-armv8-clang-14, host_profile: macos-26-armv8-clang-14, bindings: true } - { os: windows-2022, build_profile: windows-2022-msvc-1940, host_profile: windows-2022-msvc-1940 } - { os: ubuntu-24.04, build_profile: ubuntu-24.04-clang-18, host_profile: android-35-x86_64, ndk_version: 28.1.13356709 } steps: @@ -43,41 +61,53 @@ jobs: brew install ccache ccache -V + - name: setup java + if: matrix.bindings + uses: actions/setup-java@f4f1212c880fdec8162ea9a6493f4495191887b4 # v5 + with: + distribution: temurin + java-version: 21 + - name: setup python 3.14 uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: 3.14 - name: install python dependencies - run: pip install conan + run: pip install conan ${{ matrix.bindings && 'pytest' || '' }} - name: install NDK if: startsWith(matrix.host_profile, 'android') run: yes | ${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager --install 'ndk;${{ matrix.ndk_version }}' + - name: conan cache key + shell: bash + run: echo "CONAN_CACHE_KEY=$(git rev-parse HEAD:conan-odr-index)-${{ hashFiles('conanfile.py', '.github/config/conan/**') }}" >> "$GITHUB_ENV" + - name: cache conan uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 with: path: ${{ env.CONAN_HOME }} - key: conan-${{ matrix.host_profile }}-${{ env.CONAN_KEY_SUFFIX }} + key: conan-${{ env.CACHE_FLAVOR }}-${{ matrix.host_profile }}-${{ env.CONAN_KEY_SUFFIX }}-${{ env.CONAN_CACHE_KEY }} restore-keys: | - conan-${{ matrix.host_profile }}- + conan-${{ env.CACHE_FLAVOR }}-${{ matrix.host_profile }}-${{ env.CONAN_KEY_SUFFIX }}- - name: export conan-odr-index run: python conan-odr-index/scripts/conan_export_all_packages.py --selection-config conan-odr-index/defaults.yaml - name: conan config run: conan config install .github/config/conan - - name: cache ccache - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + - name: restore ccache + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 with: path: ${{ env.CCACHE_DIR }} - key: ccache-${{ matrix.host_profile }}-${{ env.CCACHE_KEY_SUFFIX }} + key: ccache-${{ env.CACHE_FLAVOR }}-${{ matrix.host_profile }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.run_id }} restore-keys: | - ccache-${{ matrix.host_profile }}- + ccache-${{ env.CACHE_FLAVOR }}-${{ matrix.host_profile }}-${{ env.CCACHE_KEY_SUFFIX }}- - name: conan install run: > conan install . + ${{ matrix.bindings && '-o "&:with_jni=True" -o "&:with_python=True"' || '' }} --profile:host '${{ matrix.host_profile }}' --profile:build '${{ matrix.build_profile }}' --build missing @@ -96,6 +126,7 @@ jobs: -DODR_WITH_WVWARE=ON -DODR_WITH_LIBMAGIC=ON -DODR_BUNDLE_ASSETS=ON + ${{ matrix.bindings && '-DODR_JNI=ON -DODR_PYTHON=ON' || '' }} - name: cmake if: runner.os == 'Windows' @@ -113,6 +144,23 @@ jobs: - name: build run: cmake --build build --config Release + - name: save ccache + if: github.ref == 'refs/heads/main' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ env.CACHE_FLAVOR }}-${{ matrix.host_profile }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.run_id }} + + # Both suites run in seconds, so they stay in this job rather than paying + # an artifact round-trip. The heavy `odr_test` still runs in `test`, which + # needs firefox and the reference-output submodule. + - name: junit + if: matrix.bindings + run: ctest --test-dir build/jni --output-on-failure + - name: pytest + if: matrix.bindings + run: ctest --test-dir build/python --output-on-failure + - name: install run: cmake --build build --target install --config Release @@ -267,6 +315,10 @@ jobs: build-test-downstream: runs-on: ${{ matrix.os }} + # Exports the full odr-index (no `--selection-config`) and resolves through + # `conan.lock`, so its dependency set differs from the `build` job's. + env: + CACHE_FLAVOR: downstream strategy: fail-fast: false matrix: @@ -296,17 +348,17 @@ jobs: - name: install python dependencies run: pip install conan - - name: install NDK - if: startsWith(matrix.host_profile, 'android') - run: yes | ${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager --install 'ndk;${{ matrix.ndk_version }}' + - name: conan cache key + shell: bash + run: echo "CONAN_CACHE_KEY=$(git rev-parse HEAD:conan-odr-index)-${{ hashFiles('conanfile.py', 'conan.lock', '.github/config/conan/**') }}" >> "$GITHUB_ENV" - name: cache conan uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 with: path: ${{ env.CONAN_HOME }} - key: conan-${{ matrix.host_profile }}-${{ env.CONAN_KEY_SUFFIX }} + key: conan-${{ env.CACHE_FLAVOR }}-${{ matrix.host_profile }}-${{ env.CONAN_KEY_SUFFIX }}-${{ env.CONAN_CACHE_KEY }} restore-keys: | - conan-${{ matrix.host_profile }}- + conan-${{ env.CACHE_FLAVOR }}-${{ matrix.host_profile }}-${{ env.CONAN_KEY_SUFFIX }}- - name: export conan-odr-index run: python conan-odr-index/scripts/conan_export_all_packages.py @@ -316,13 +368,13 @@ jobs: - name: conan odrcore run: conan export . --name odrcore --version 0.0.0 - - name: cache ccache - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + - name: restore ccache + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 with: path: ${{ env.CCACHE_DIR }} - key: ccache-${{ matrix.host_profile }}-${{ env.CCACHE_KEY_SUFFIX }} + key: ccache-${{ env.CACHE_FLAVOR }}-${{ matrix.host_profile }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.run_id }} restore-keys: | - ccache-${{ matrix.host_profile }}- + ccache-${{ env.CACHE_FLAVOR }}-${{ matrix.host_profile }}-${{ env.CCACHE_KEY_SUFFIX }}- - name: conan install run: > @@ -350,6 +402,13 @@ jobs: - name: build run: cmake --build test/downstream/build --config Release + - name: save ccache + if: github.ref == 'refs/heads/main' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ env.CACHE_FLAVOR }}-${{ matrix.host_profile }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.run_id }} + - name: run if: matrix.build_profile == matrix.host_profile && (runner.os == 'Linux' || runner.os == 'macOS') run: test/downstream/build/odr-test-downstream diff --git a/.github/workflows/jni.yml b/.github/workflows/jni.yml deleted file mode 100644 index d0ba5c69..00000000 --- a/.github/workflows/jni.yml +++ /dev/null @@ -1,102 +0,0 @@ -name: jni - -on: - push: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -env: - CCACHE_DIR: ${{ github.workspace }}/.ccache - CCACHE_MAXSIZE: 1G - CCACHE_KEY_SUFFIX: r1 - CONAN_HOME: ${{ github.workspace }}/.conan2 - CONAN_KEY_SUFFIX: r1 - -jobs: - build-test: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - include: - - { os: ubuntu-24.04, build_profile: ubuntu-24.04-clang-18, host_profile: ubuntu-24.04-clang-18 } - - { os: macos-26, build_profile: macos-26-armv8-clang-14, host_profile: macos-26-armv8-clang-14 } - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - name: checkout conan-odr-index - run: git submodule update --init --depth 1 conan-odr-index - - - name: ubuntu install ccache - if: runner.os == 'Linux' - run: | - sudo apt install ccache - ccache -V - - name: macos install ccache - if: runner.os == 'macOS' - run: | - brew install ccache - ccache -V - - - name: setup java - uses: actions/setup-java@f4f1212c880fdec8162ea9a6493f4495191887b4 # v5 - with: - distribution: temurin - java-version: 21 - - - name: setup python 3.14 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 - with: - python-version: 3.14 - - name: install python dependencies - run: pip install conan - - - name: cache conan - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 - with: - path: ${{ env.CONAN_HOME }} - key: conan-jni-${{ matrix.host_profile }}-${{ env.CONAN_KEY_SUFFIX }} - restore-keys: | - conan-jni-${{ matrix.host_profile }}- - conan-${{ matrix.host_profile }}- - - - name: export conan-odr-index - run: python conan-odr-index/scripts/conan_export_all_packages.py --selection-config conan-odr-index/defaults.yaml - - name: conan config - run: conan config install .github/config/conan - - - name: cache ccache - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 - with: - path: ${{ env.CCACHE_DIR }} - key: ccache-jni-${{ matrix.host_profile }}-${{ env.CCACHE_KEY_SUFFIX }} - restore-keys: | - ccache-jni-${{ matrix.host_profile }}- - ccache-${{ matrix.host_profile }}- - - - name: conan install - run: > - conan install . - -o '&:with_jni=True' - --profile:host '${{ matrix.host_profile }}' - --profile:build '${{ matrix.build_profile }}' - --build missing - - - name: cmake - run: > - cmake -B build -S . - -DCMAKE_TOOLCHAIN_FILE="conan_toolchain.cmake" - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_CXX_FLAGS="-Werror" - -DODR_JNI=ON - -DODR_CLI=OFF - -DODR_TEST=ON - - - name: build - run: cmake --build build --target odr_jni odr_java odr_java_tests --config Release - - - name: junit - run: ctest --test-dir build/jni --output-on-failure diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 538dca83..51c9f4d8 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -10,6 +10,7 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +# See the cache-key comment in `build_test.yml` for why the keys look like this. env: CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 1G @@ -34,88 +35,15 @@ jobs: - name: check python formatting run: black --check --diff python - build-test: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - include: - - { os: ubuntu-24.04, build_profile: ubuntu-24.04-clang-18, host_profile: ubuntu-24.04-clang-18 } - - { os: macos-26, build_profile: macos-26-armv8-clang-14, host_profile: macos-26-armv8-clang-14 } - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - name: checkout conan-odr-index - run: git submodule update --init --depth 1 conan-odr-index - - - name: ubuntu install ccache - if: runner.os == 'Linux' - run: | - sudo apt install ccache - ccache -V - - name: macos install ccache - if: runner.os == 'macOS' - run: | - brew install ccache - ccache -V - - - name: setup python 3.14 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 - with: - python-version: 3.14 - - name: install python dependencies - run: pip install conan pytest - - - name: cache conan - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 - with: - path: ${{ env.CONAN_HOME }} - key: conan-python-${{ matrix.host_profile }}-${{ env.CONAN_KEY_SUFFIX }} - restore-keys: | - conan-python-${{ matrix.host_profile }}- - conan-${{ matrix.host_profile }}- - - - name: export conan-odr-index - run: python conan-odr-index/scripts/conan_export_all_packages.py --selection-config conan-odr-index/defaults.yaml - - name: conan config - run: conan config install .github/config/conan - - - name: cache ccache - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 - with: - path: ${{ env.CCACHE_DIR }} - key: ccache-python-${{ matrix.host_profile }}-${{ env.CCACHE_KEY_SUFFIX }} - restore-keys: | - ccache-python-${{ matrix.host_profile }}- - ccache-${{ matrix.host_profile }}- - - - name: conan install - run: > - conan install . - -o '&:with_python=True' - --profile:host '${{ matrix.host_profile }}' - --profile:build '${{ matrix.build_profile }}' - --build missing - - - name: cmake - run: > - cmake -B build -S . - -DCMAKE_TOOLCHAIN_FILE="conan_toolchain.cmake" - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_CXX_FLAGS="-Werror" - -DODR_PYTHON=ON - -DODR_CLI=OFF - -DODR_TEST=OFF - - - name: build - run: cmake --build build --target pyodr_core --config Release - - - name: pytest - run: PYTHONPATH=build/python python -m pytest python/tests -v - + # The bindings themselves are built and tested by the `build` job in + # `build_test.yml`, out of the same configure as the core. What stays here is + # everything that needs the wheel's own, reduced dependency set. wheels: runs-on: ${{ matrix.os }} + # Builds the reduced dependency set that `pyproject.toml` pins (no + # pdf2htmlEX/wvWare/libmagic), so it cannot share a cache with `build_test`. + env: + CACHE_FLAVOR: wheel strategy: fail-fast: false matrix: @@ -151,27 +79,30 @@ jobs: - name: install python dependencies run: pip install conan build pytest + - name: conan cache key + shell: bash + run: echo "CONAN_CACHE_KEY=$(git rev-parse HEAD:conan-odr-index)-${{ hashFiles('conanfile.py', 'pyproject.toml', '.github/config/conan/**') }}" >> "$GITHUB_ENV" + - name: cache conan uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 with: path: ${{ env.CONAN_HOME }} - key: conan-wheel-${{ matrix.host_profile }}-${{ env.CONAN_KEY_SUFFIX }} + key: conan-${{ env.CACHE_FLAVOR }}-${{ matrix.host_profile }}-${{ env.CONAN_KEY_SUFFIX }}-${{ env.CONAN_CACHE_KEY }} restore-keys: | - conan-wheel-${{ matrix.host_profile }}- + conan-${{ env.CACHE_FLAVOR }}-${{ matrix.host_profile }}-${{ env.CONAN_KEY_SUFFIX }}- - name: export conan-odr-index run: python conan-odr-index/scripts/conan_export_all_packages.py --selection-config conan-odr-index/defaults.yaml - name: conan config run: conan config install .github/config/conan - - name: cache ccache - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + - name: restore ccache + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 with: path: ${{ env.CCACHE_DIR }} - key: ccache-wheel-${{ matrix.host_profile }}-${{ env.CCACHE_KEY_SUFFIX }} + key: ccache-${{ env.CACHE_FLAVOR }}-${{ matrix.host_profile }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.run_id }} restore-keys: | - ccache-wheel-${{ matrix.host_profile }}- - ccache-${{ matrix.host_profile }}- + ccache-${{ env.CACHE_FLAVOR }}-${{ matrix.host_profile }}-${{ env.CCACHE_KEY_SUFFIX }}- # Match the dependency set that pyproject.toml enables (no pdf2htmlEX, # wvWare, libmagic — their runtime data cannot ship inside the wheel). @@ -191,6 +122,13 @@ jobs: env: CMAKE_ARGS: -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/conan_toolchain.cmake + - name: save ccache + if: github.ref == 'refs/heads/main' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ env.CACHE_FLAVOR }}-${{ matrix.host_profile }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.run_id }} + # PyPI rejects plain `linux_*` platform tags; auditwheel grafts the wheel # onto the matching manylinux tag (the conan deps are linked statically, # so there should be no external libraries to vendor). @@ -236,7 +174,7 @@ jobs: if-no-files-found: error publish: - needs: [build-test, wheels, sdist] + needs: [wheels, sdist] runs-on: ubuntu-24.04 if: github.event_name == 'release' && github.event.action == 'published' permissions: @@ -259,7 +197,7 @@ jobs: # conan-generated toolchain plus the conan-odr-index recipes, see # python/README.md); it is published for completeness. pypi: - needs: [build-test, wheels, sdist] + needs: [wheels, sdist] runs-on: ubuntu-24.04 if: github.event_name == 'release' && github.event.action == 'published' environment: pypi