From 02315bd891e95cead7724577ce32bcd8c39d229d Mon Sep 17 00:00:00 2001 From: "Harlow, Jordan" Date: Fri, 30 Oct 2026 08:48:35 -0600 Subject: [PATCH] Build and test conda packages and wheels for Intel Python CI Adds what Intel Python CI (gh-ci) needs: a ci/ directory and .github/workflows/intel-ci.yml. The workflow builds conda-recipe-cf with conda-build and the wheels with cibuildwheel, for every platform and Python version the recipe and pyproject.toml declare, and tests each package in a clean environment with ci/test.py. Intel's Jenkins tests them again, scans them, runs ci/benchmark.py on the newest Linux package, and publishes a release tag's tested packages as its GitHub Release. conda-recipe-cf/ is now the only recipe: conda-recipe/ and conda-package.yml, which built it against Intel's channel and copied wheels out through WHEELS_OUTPUT_FOLDER, are removed. conda-package-cf.yml reports the same check names, so the required checks still run. conda_build_config.yaml now lists the Python versions. The wheels link MKL's layers and the Intel OpenMP runtime, so their run-time requirements on mkl and intel-openmp come from [tool.gh-ci] wheel-requires in pyproject.toml, which ci/build.py adds to each wheel: in [project] they would fail the recipe's pip check, as conda-forge's mkl has no pip metadata. --- .github/AGENTS.md | 4 +- .github/workflows/conda-package.yml | 473 ----------------- .github/workflows/coverity.yml | 7 +- .github/workflows/intel-ci.yml | 187 +++++++ AGENTS.md | 5 +- README.md | 2 +- ci/README.md | 33 ++ ci/benchmark.py | 90 ++++ ci/build.py | 343 +++++++++++++ ci/config.yml | 12 + ci/hooks.py | 440 ++++++++++++++++ ci/install_icx.py | 62 +++ ci/matrix.py | 653 ++++++++++++++++++++++++ ci/test.py | 53 ++ conda-recipe-cf/AGENTS.md | 22 +- conda-recipe-cf/conda_build_config.yaml | 9 + conda-recipe-cf/meta.yaml | 10 + conda-recipe/AGENTS.md | 30 -- conda-recipe/bld.bat | 25 - conda-recipe/build.sh | 41 -- conda-recipe/conda_build_config.yaml | 20 - conda-recipe/meta.yaml | 67 --- meson.build | 8 +- mkl_umath/tests/AGENTS.md | 2 +- pyproject.toml | 28 +- 25 files changed, 1939 insertions(+), 687 deletions(-) delete mode 100644 .github/workflows/conda-package.yml create mode 100644 .github/workflows/intel-ci.yml create mode 100644 ci/README.md create mode 100644 ci/benchmark.py create mode 100644 ci/build.py create mode 100644 ci/config.yml create mode 100644 ci/hooks.py create mode 100644 ci/install_icx.py create mode 100644 ci/matrix.py create mode 100644 ci/test.py delete mode 100644 conda-recipe/AGENTS.md delete mode 100644 conda-recipe/bld.bat delete mode 100644 conda-recipe/build.sh delete mode 100644 conda-recipe/conda_build_config.yaml delete mode 100644 conda-recipe/meta.yaml diff --git a/.github/AGENTS.md b/.github/AGENTS.md index 47576d02..9d80938d 100644 --- a/.github/AGENTS.md +++ b/.github/AGENTS.md @@ -3,8 +3,8 @@ CI/CD workflows and repo automation. ## Workflows (source of truth) -- `conda-package.yml` — Intel channel conda build/test pipeline - `conda-package-cf.yml` — conda-forge-oriented build/test pipeline +- `intel-ci.yml` — Intel Python CI: builds and tests the conda packages of `conda-recipe-cf` and the wheels, for the platforms and Python versions the recipe and `pyproject.toml` declare (`ci/matrix.py`); see `ci/README.md` - `build-with-clang.yml` — Intel clang compatibility checks - `build-with-standard-clang.yml` — standard clang compatibility checks - `build_pip.yml` — pip build pipeline with pre-release NumPy @@ -12,5 +12,5 @@ CI/CD workflows and repo automation. - `openssf-scorecard.yml` — security scanning ## Policy -- Treat workflow YAML as canonical for platform/Python matrices. +- Treat workflow YAML as canonical for platform/Python matrices, except `intel-ci.yml`, whose conda matrix is `conda-recipe-cf/conda_build_config.yaml` and whose wheel matrix is the Python classifiers in `pyproject.toml`. - Avoid doc claims about CI coverage unless present in workflow config. diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml deleted file mode 100644 index 0c3e50af..00000000 --- a/.github/workflows/conda-package.yml +++ /dev/null @@ -1,473 +0,0 @@ -name: Conda package - -on: - push: - branches: - - main - pull_request: - -permissions: - contents: read - -env: - PACKAGE_NAME: mkl_umath - MODULE_NAME: mkl_umath - TEST_ENV_NAME: test_mkl_umath - VER_SCRIPT1: "import json; f = open('ver.json', 'r'); j = json.load(f); f.close(); d = j['mkl_umath'][0];" - VER_SCRIPT2: "print('='.join((d[s] for s in ('version', 'build'))))" - CONDA_BUILD_VERSION: 26.3.0 - -jobs: - build_linux: - runs-on: ubuntu-latest - strategy: - matrix: - include: - - python: "3.10" - numpy: "2.2" - - python: "3.11" - numpy: "2.3" - - python: "3.12" - numpy: "2.3" - - python: "3.13" - numpy: "2.3" - - python: "3.14" - numpy: "2.3" - python_spec: "3.14.* *_cp314" - python_tag: "3.14" - - python: "3.14" - numpy: "2.3" - python_spec: "3.14.* *_cp314t" - python_tag: "3.14t" - - python: "3.15" - numpy: "2.5" - python_spec: "3.15.* *_cp315" - python_tag: "3.15" - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@d07a454dad7609a92316b57b23c9ccfd4f59af66 # 0.13.1 - with: - access_token: ${{ github.token }} - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 0 # Increase to reset cache - with: - path: ~/.conda/pkgs - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}-${{hashFiles('**/meta.yaml') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: latest - activate-environment: "" - pkgs-dirs: ~/.conda/pkgs - - - name: Update conda - run: | - conda update -n base --all - - - name: Install conda-build - run: conda install -n base conda-build=${{ env.CONDA_BUILD_VERSION }} -c conda-forge --override-channels - - - name: Show Conda info - run: | - conda info --all - - - name: List base environment packages - run: | - conda list -n base - - - name: Store conda paths as envs - run: | - echo "CONDA_BLD=${CONDA}/conda-bld/linux-64" >> "$GITHUB_ENV" - echo "WHEELS_OUTPUT_FOLDER=$GITHUB_WORKSPACE/" >> "$GITHUB_ENV" - - - name: Build conda package - run: | - CHANNELS=(-c "https://software.repos.intel.com/python/conda" -c "conda-forge" --override-channels) - VERSIONS=(--python "${{ matrix.python_spec || matrix.python }}" --numpy "${{ matrix.numpy }}") - TEST=(--no-test) - - conda build \ - "${TEST[@]}" \ - "${VERSIONS[@]}" \ - "${CHANNELS[@]}" \ - conda-recipe - - - name: Upload artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python_tag || matrix.python }} - path: ${{ env.CONDA_BLD }}/${{ env.PACKAGE_NAME }}-*.conda - - - name: Upload wheels artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Wheels Python ${{ matrix.python_tag || matrix.python }} - path: ${{ env.WHEELS_OUTPUT_FOLDER }}${{ env.PACKAGE_NAME }}-*.whl - - test_linux: - needs: build_linux - runs-on: ${{ matrix.runner }} - - strategy: - matrix: - python: ["3.10", "3.11", "3.12", "3.13"] - numpy: ['numpy">=2"'] - experimental: [false] - runner: [ubuntu-latest] - include: - - python: "3.14" - python_spec: "3.14.* *_cp314" - python_tag: "3.14" - numpy: 'numpy">=2"' - experimental: false - runner: ubuntu-latest - - python: "3.14" - python_spec: "3.14.* *_cp314t" - python_tag: "3.14t" - numpy: 'numpy">=2"' - experimental: false - runner: ubuntu-latest - - python: "3.15" - python_spec: "3.15.* *_cp315" - python_tag: "3.15" - numpy: 'numpy">=2"' - experimental: false - runner: ubuntu-latest - continue-on-error: ${{ matrix.experimental }} - env: - CHANNELS: -c https://software.repos.intel.com/python/conda -c conda-forge --override-channels - - steps: - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python_tag || matrix.python }} - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: latest - activate-environment: "" - pkgs-dirs: ~/.conda/pkgs - - name: Update conda - run: | - conda update -n base --all - - name: Install conda-index - run: | - conda install -n base conda-index -c conda-forge --override-channels - - name: Show Conda info - run: | - conda info --all - - name: List base environment packages - run: | - conda list -n base - - name: Create conda channel - run: | - mkdir -p "$GITHUB_WORKSPACE/channel/linux-64" - mv "${PACKAGE_NAME}"-*.conda "$GITHUB_WORKSPACE/channel/linux-64" - conda index "$GITHUB_WORKSPACE/channel" - # Test channel - conda search "$PACKAGE_NAME" -c "$GITHUB_WORKSPACE/channel" --override-channels - - name: Test conda channel - run: | - conda search "$PACKAGE_NAME" -c "$GITHUB_WORKSPACE"/channel --override-channels --info --json > "$GITHUB_WORKSPACE"/ver.json - cat "$GITHUB_WORKSPACE"/ver.json - - name: Collect dependencies - run: | - CHANNELS=(-c "$GITHUB_WORKSPACE/channel" -c "https://software.repos.intel.com/python/conda" -c "conda-forge" --override-channels) - PACKAGE_VERSION="$(python -c "${VER_SCRIPT1} ${VER_SCRIPT2}")" - conda create -n "${{ env.TEST_ENV_NAME }}" "${PACKAGE_NAME}=${PACKAGE_VERSION}" "python=${{ matrix.python_spec || matrix.python }}" ${{ matrix.numpy }} "${CHANNELS[@]}" --only-deps --dry-run > lockfile - - name: Display lockfile - run: cat lockfile - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 0 # Increase to reset cache - with: - path: ~/.conda/pkgs - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}-${{hashFiles('lockfile') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - - name: Install mkl_umath - run: | - CHANNELS=(-c "$GITHUB_WORKSPACE/channel" -c "https://software.repos.intel.com/python/conda" -c "conda-forge" --override-channels) - PACKAGE_VERSION="$(python -c "${VER_SCRIPT1} ${VER_SCRIPT2}")" - conda create -n "${{ env.TEST_ENV_NAME }}" "python=${{ matrix.python_spec || matrix.python }}" ${{ matrix.numpy }} "$PACKAGE_NAME=${PACKAGE_VERSION}" pytest "${CHANNELS[@]}" - # Test installed packages - conda list -n "${{ env.TEST_ENV_NAME }}" - - - name: Smoke test - run: | - source "$CONDA/etc/profile.d/conda.sh" - conda activate "${{ env.TEST_ENV_NAME }}" - python -c "import mkl_umath, numpy as np; mkl_umath.patch_numpy_umath(); np.sin(np.linspace(0, 1, num=10**6));" - - - name: Run tests - run: | - source "$CONDA/etc/profile.d/conda.sh" - conda activate "${{ env.TEST_ENV_NAME }}" - pytest -v --pyargs ${{ env.MODULE_NAME }} - - build_windows: - runs-on: windows-latest - - strategy: - matrix: - include: - - python: "3.10" - numpy: "2.2" - - python: "3.11" - numpy: "2.3" - - python: "3.12" - numpy: "2.3" - - python: "3.13" - numpy: "2.3" - - python: "3.14" - numpy: "2.3" - python_spec: "3.14.* *_cp314" - python_tag: "3.14" - - python: "3.14" - numpy: "2.3" - python_spec: "3.14.* *_cp314t" - python_tag: "3.14t" - - python: "3.15" - numpy: "2.5" - python_spec: "3.15.* *_cp315" - python_tag: "3.15" - env: - conda-bld: C:\Miniconda\conda-bld\win-64\ - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@d07a454dad7609a92316b57b23c9ccfd4f59af66 # 0.13.1 - with: - access_token: ${{ github.token }} - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - auto-update-conda: true - miniforge-version: latest - activate-environment: build - channels: conda-forge - python-version: ${{ matrix.python }} - conda-build-version: ${{ env.CONDA_BUILD_VERSION }} - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 3 # Increase to reset cache - with: - path: ~/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}-${{hashFiles('**/meta.yaml') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - - name: Store conda paths as envs - shell: bash -l {0} - run: | - echo "CONDA_BLD=$CONDA/conda-bld/win-64/" | tr "\\\\" '/' >> "$GITHUB_ENV" - echo "WHEELS_OUTPUT_FOLDER=$GITHUB_WORKSPACE\\" >> "$GITHUB_ENV" - - - name: Install conda build - run: | - conda activate - conda install -y conda-build - conda list -n base - - - name: Build conda package - run: | - conda activate - conda build --no-test --python "${{ matrix.python_spec || matrix.python }}" --numpy ${{ matrix.numpy }} -c https://software.repos.intel.com/python/conda -c conda-forge --override-channels conda-recipe - - - name: Upload artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python_tag || matrix.python }} - path: ${{ env.CONDA_BLD }}${{ env.PACKAGE_NAME }}-*.conda - - - name: Upload wheels artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Wheels Python ${{ matrix.python_tag || matrix.python }} - path: ${{ env.WHEELS_OUTPUT_FOLDER }}${{ env.PACKAGE_NAME }}-*.whl - - test_windows: - needs: build_windows - runs-on: ${{ matrix.runner }} - defaults: - run: - shell: cmd /C CALL {0} - strategy: - matrix: - python: ["3.10", "3.11", "3.12", "3.13"] - numpy: ['numpy">=2"'] - experimental: [false] - runner: [windows-latest] - include: - - python: "3.14" - python_spec: "3.14.* *_cp314" - python_tag: "3.14" - numpy: 'numpy">=2"' - experimental: false - runner: windows-latest - - python: "3.14" - python_spec: "3.14.* *_cp314t" - python_tag: "3.14t" - numpy: 'numpy">=2"' - experimental: false - runner: windows-latest - - python: "3.15" - python_spec: "3.15.* *_cp315" - python_tag: "3.15" - numpy: 'numpy">=2"' - experimental: false - runner: windows-latest - continue-on-error: ${{ matrix.experimental }} - env: - workdir: '${{ github.workspace }}' - CHANNELS: -c https://software.repos.intel.com/python/conda -c conda-forge --override-channels - - steps: - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python_tag || matrix.python }} - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - auto-update-conda: true - miniforge-version: latest - activate-environment: ${{ env.TEST_ENV_NAME }} - channels: conda-forge - conda-remove-defaults: "true" - python-version: ${{ matrix.python }} - - - name: Install conda-index - run: | - conda install -n base conda-index - - - name: Show Conda info - run: | - conda info --all - - - name: List base environment packages - run: | - conda list -n base - - - name: Create conda channel with the artifact bit - shell: cmd /C CALL {0} - run: | - echo ${{ env.workdir }} - mkdir ${{ env.workdir }}\channel - mkdir ${{ env.workdir }}\channel\win-64 - move ${{ env.PACKAGE_NAME }}-*.conda ${{ env.workdir }}\channel\win-64 - dir ${{ env.workdir }}\channel\win-64 - - - name: Index the channel - shell: cmd /C CALL {0} - run: | - conda index ${{ env.workdir }}\channel - - - name: Dump mkl_umath version info from created channel to STDOUT - shell: cmd /C CALL {0} - run: | - conda search ${{ env.PACKAGE_NAME }} -c ${{ env.workdir }}/channel --override-channels --info --json - - - name: Dump mkl_umath version info from created channel into ver.json - shell: cmd /C CALL {0} - run: | - conda search ${{ env.PACKAGE_NAME }} -c ${{ env.workdir }}/channel --override-channels --info --json > ${{ env.workdir }}\ver.json - - - name: Output content of workdir - shell: pwsh - run: Get-ChildItem -Path ${{ env.workdir }} - - - name: Output content of produced ver.json - shell: pwsh - run: Get-Content -Path ${{ env.workdir }}\ver.json - - - name: Collect dependencies - shell: cmd /C CALL {0} - run: | - @ECHO ON - IF NOT EXIST ver.json ( - copy /Y ${{ env.workdir }}\ver.json . - ) - SET "SCRIPT=%VER_SCRIPT1% %VER_SCRIPT2%" - FOR /F "tokens=* USEBACKQ" %%F IN (`python -c "%SCRIPT%"`) DO ( - SET PACKAGE_VERSION=%%F - ) - conda install -n ${{ env.TEST_ENV_NAME }} ${{ env.PACKAGE_NAME }}=%PACKAGE_VERSION% "python=${{ matrix.python_spec || matrix.python }}" ${{ matrix.numpy }} -c ${{ env.workdir }}/channel ${{ env.CHANNELS }} --only-deps --dry-run > lockfile - - - name: Display lockfile content - shell: pwsh - run: Get-Content -Path .\lockfile - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 0 # Increase to reset cache - with: - path: ~/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}-${{hashFiles('lockfile') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - # add intel-openmp as an explicit dependency - # to avoid it being missed when package version is specified exactly - - name: Install mkl_umath - shell: cmd /C CALL {0} - run: | - @ECHO ON - IF NOT EXIST ver.json ( - copy /Y ${{ env.workdir }}\ver.json . - ) - set "SCRIPT=%VER_SCRIPT1% %VER_SCRIPT2%" - FOR /F "tokens=* USEBACKQ" %%F IN (`python -c "%SCRIPT%"`) DO ( - SET PACKAGE_VERSION=%%F - ) - SET "TEST_DEPENDENCIES=pytest pytest-cov" - SET "WORKAROUND_DEPENDENCIES=intel-openmp" - SET "DEPENDENCIES=%TEST_DEPENDENCIES% %WORKAROUND_DEPENDENCIES%" - conda install -n ${{ env.TEST_ENV_NAME }} ${{ env.PACKAGE_NAME }}=%PACKAGE_VERSION% %DEPENDENCIES% "python=${{ matrix.python_spec || matrix.python }}" ${{ matrix.numpy }} -c ${{ env.workdir }}/channel ${{ env.CHANNELS }} - - - name: Report content of test environment - shell: cmd /C CALL {0} - run: | - CALL conda activate - echo "Value of CONDA environment variable was: " %CONDA% - echo "Value of CONDA_PREFIX environment variable was: " %CONDA_PREFIX% - CALL conda info && CALL conda list -n ${{ env.TEST_ENV_NAME }} - - - name: Smoke test - shell: cmd /C CALL {0} - run: | - @ECHO ON - CALL conda activate ${{ env.TEST_ENV_NAME }} - python -c "import mkl_umath, numpy as np; mkl_umath.patch_numpy_umath(); np.sin(np.linspace(0, 1, num=10**6));" - - - name: Run tests - shell: cmd /C CALL {0} - run: | - CALL conda activate ${{ env.TEST_ENV_NAME }} - python -m pytest -v -s --pyargs ${{ env.MODULE_NAME }} diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index 5f243cda..2c92bfce 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -79,11 +79,8 @@ jobs: run: | # shellcheck disable=SC1091 source "${ONEAPI_ROOT}/setvars.sh" - # Build with icx to match the shipped conda packages (both - # conda-recipe and conda-recipe-cf use CC=icx). meson.build only - # applies -fveclib=SVML, -fvectorize, -fimf-precision=high and - # -fprotect-parens when the compiler id is Intel, so gcc would not - # exercise that branch. + # Build with icx, as conda-recipe-cf does: meson.build adds its SVML + # and precision flags only for an Intel compiler. export CC="${CMPLR_ROOT}/bin/icx" # icx is not auto-configured by the Coverity front-end, so register it # explicitly first. Two constraints on this command: diff --git a/.github/workflows/intel-ci.yml b/.github/workflows/intel-ci.yml new file mode 100644 index 00000000..b81d5000 --- /dev/null +++ b/.github/workflows/intel-ci.yml @@ -0,0 +1,187 @@ +# Intel Python CI (gh-ci): builds and tests the conda packages and wheels for Intel's Jenkins, see ci/README.md +name: Intel Python CI + +on: + push: + branches: [main] + tags: ['*'] + pull_request: + +permissions: + contents: read + +concurrency: + group: intel-ci-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + matrix: + name: Platforms and Python versions + runs-on: ${{ vars.GH_CI_RUNNER || 'ubuntu-latest' }} + timeout-minutes: 10 + outputs: + matrix: ${{ steps.declared.outputs.matrix }} + wheel: ${{ steps.declared.outputs.wheel }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Read the declared platforms and Python versions + id: declared + run: | + python ci/matrix.py --event "$GITHUB_EVENT_NAME" --github-output matrix + python ci/matrix.py --system wheel --event "$GITHUB_EVENT_NAME" --github-output wheel + + # macOS runs on Intel runners, as MKL has no Apple silicon build; Jenkins ignores it. + build: + name: ${{ matrix.name }} Python ${{ matrix.python }} + needs: matrix + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The recipe takes its version from git describe. + fetch-depth: 0 + + # Miniforge 26.9.4-0 fails setup-miniconda's conda init on Windows runners; 26.7.2-0 works. + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install conda-build + run: conda install -y -q -n base --override-channels -c conda-forge conda-build + + - name: Build and run the recipe tests + env: + PYTHON_VERSION: ${{ matrix.python }} + run: python ci/build.py --python "$PYTHON_VERSION" --output "$RUNNER_TEMP/output" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: mkl_umath ${{ matrix.name }} Python ${{ matrix.python }} + path: ${{ runner.temp }}/output/*.conda + if-no-files-found: error + # Jenkins downloads them within minutes of the run; tags keep them 30 days. + retention-days: ${{ startsWith(github.ref, 'refs/tags/') && 30 || 3 }} + + test: + name: Test ${{ matrix.name }} Python ${{ matrix.python }} + needs: [matrix, build] + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: mkl_umath ${{ matrix.name }} Python ${{ matrix.python }} + path: ${{ runner.temp }}/artifacts + + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install the package and run ci/test.py + env: + PYTHON_VERSION: ${{ matrix.python }} + EVENT: ${{ startsWith(github.ref, 'refs/tags/') && 'tag' || github.event_name == 'pull_request' && 'pr' || 'push' }} + COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + run: >- + python ci/hooks.py --python "$PYTHON_VERSION" --artifacts "$RUNNER_TEMP/artifacts" + --work "$RUNNER_TEMP/hooks" --event "$EVENT" --commit "$COMMIT" + + build-wheel: + name: ${{ matrix.name }} Wheels Python ${{ matrix.python }} + needs: matrix + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.wheel) }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Install cibuildwheel + run: python -m pip install --no-input cibuildwheel==4.3.5 + + - name: Build the wheel + env: + PYTHON_VERSION: ${{ matrix.python }} + run: python ci/build.py --system wheel --python "$PYTHON_VERSION" --output "$RUNNER_TEMP/output" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: mkl_umath ${{ matrix.name }} Wheels Python ${{ matrix.python }} + path: ${{ runner.temp }}/output/*.whl + if-no-files-found: error + retention-days: ${{ startsWith(github.ref, 'refs/tags/') && 30 || 3 }} + + test-wheel: + name: Test ${{ matrix.name }} Wheels Python ${{ matrix.python }} + needs: [matrix, build-wheel] + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.wheel) }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: mkl_umath ${{ matrix.name }} Wheels Python ${{ matrix.python }} + path: ${{ runner.temp }}/artifacts + + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install the wheel and run ci/test.py + env: + PYTHON_VERSION: ${{ matrix.python }} + EVENT: ${{ startsWith(github.ref, 'refs/tags/') && 'tag' || github.event_name == 'pull_request' && 'pr' || 'push' }} + COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + run: >- + python ci/hooks.py --kind wheel --python "$PYTHON_VERSION" --artifacts "$RUNNER_TEMP/artifacts" + --work "$RUNNER_TEMP/hooks" --event "$EVENT" --commit "$COMMIT" diff --git a/AGENTS.md b/AGENTS.md index 265f9110..c5b990e4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,8 +43,8 @@ CC=icx pip install --no-deps --no-build-isolation . # clang is also supported i - **Platforms:** Linux, Windows - **Python versions:** 3.10, 3.11, 3.12, 3.13, 3.14, 3.15 - **Workflows:** `.github/workflows/` - - `conda-package.yml` — main conda build/test pipeline - `conda-package-cf.yml` — conda-forge-oriented build/test pipeline + - `intel-ci.yml` — Intel Python CI: builds and tests the conda packages and the wheels; see `ci/README.md` - `build_pip.yml` — validates pip build with pre-release NumPy - `build-with-clang.yml` — Intel clang compatibility check - `build-with-standard-clang.yml` — standard clang compatibility check @@ -97,8 +97,7 @@ Below directories have local `AGENTS.md` for deeper context: - `mkl_umath/AGENTS.md` — Python API and code generation - `mkl_umath/src/AGENTS.md` — C/Cython implementation layer - `mkl_umath/tests/AGENTS.md` — unit tests and validation -- `conda-recipe/AGENTS.md` — Intel channel conda packaging -- `conda-recipe-cf/AGENTS.md` — conda-forge compatible recipe +- `conda-recipe-cf/AGENTS.md` — the conda recipe - `_vendored/AGENTS.md` — vendored NumPy utilities --- diff --git a/README.md b/README.md index 9e6cafad..a186832e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Conda package](https://github.com/IntelPython/mkl_umath/actions/workflows/conda-package.yml/badge.svg)](https://github.com/IntelPython/mkl_umath/actions/workflows/conda-package.yml) +[![Conda package using conda-forge](https://github.com/IntelPython/mkl_umath/actions/workflows/conda-package-cf.yml/badge.svg)](https://github.com/IntelPython/mkl_umath/actions/workflows/conda-package-cf.yml) [![Build using pip and pre-release NumPy](https://github.com/IntelPython/mkl_umath/actions/workflows/build_pip.yml/badge.svg)](https://github.com/IntelPython/mkl_umath/actions/workflows/build_pip.yml) [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/IntelPython/mkl_umath/badge)](https://securityscorecards.dev/viewer/?uri=github.com/IntelPython/mkl_umath) [![Coverity Scan Build Status](https://scan.coverity.com/projects/33201/badge.svg)](https://scan.coverity.com/projects/intelpython-mkl_umath) diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 00000000..f75256ef --- /dev/null +++ b/ci/README.md @@ -0,0 +1,33 @@ +# How Intel Python CI tests mkl_umath + +Intel Python CI (gh-ci) builds and tests mkl_umath's conda packages and wheels in GitHub Actions. Intel's Jenkins then tests each package again on Intel machines and scans it, and on a release tag publishes the packages it tested as the tag's GitHub Release. Everything it needs from this repository is in this directory, one workflow, the recipe, and `pyproject.toml`. + +| File | Role | +|---|---| +| `../conda-recipe-cf/` | The conda recipe. `conda_build_config.yaml` lists the Python versions. The recipe builds on Linux, Windows and Intel macOS unless a platform-only skip in `meta.yaml`, such as `skip: true # [win]`, rules one out. | +| `../pyproject.toml` | What the wheels are built from. Its Python classifiers list the wheels' Python versions, and its Operating System classifiers their platforms. `[tool.cibuildwheel]` holds the wheel build settings, and `[tool.gh-ci]` holds the run-time requirements the wheels need on top of `[project]`'s, which the conda packages get from the recipe. | +| `../.github/workflows/intel-ci.yml` | One job reads both matrices. One job per platform and Python version builds and uploads `mkl_umath Python `, and one builds `mkl_umath Wheels Python `. Then one job per package tests it. | +| `matrix.py` | Reads the platforms and Python versions from the recipe, or with `--system wheel` from `pyproject.toml`. A pull request gets the oldest and the newest regular versions. | +| `build.py` | Builds one Python version: with conda-build, which runs the recipe's own tests, or with `--system wheel` with cibuildwheel, in a manylinux container on Linux. | +| `hooks.py` | The workflow's test jobs: installs one package into a clean environment with the requirements in `config.yml`, and runs `test.py` the way Jenkins does. A conda package's environment comes from conda-forge; a wheel's has only Python from conda-forge and everything else from PyPI. | +| `install_icx.py` | Installs Intel's icx, which the conda recipe builds with too, for the Windows wheel builds: `[tool.cibuildwheel.windows]` in `../pyproject.toml` runs it first. The Linux wheel builds get icx from Intel's yum repository. | +| `config.yml` | Tells gh-ci which workflow builds the packages, what its artifacts are called, and what the tests and benchmarks need. | +| `test.py` | Runs the installed package's test suite with pytest, in Actions and again in Jenkins. | +| `benchmark.py` | Runs the asv suite in `../benchmarks/` against the installed package, in Jenkins only, on a bare-metal benchmarking machine. Jenkins requires it because `../benchmarks/` exists. | + +The scripts need only the Python standard library and are the same in every repository gh-ci tests, except that `matrix.py` here looks for `conda-recipe-cf/` first. Scanning is not the repository's to define: Jenkins runs its own scans. The recipe skips macOS, as `conda-forge/mkl_umath-feedstock` does. + +**A pull request into `main`.** Actions builds the oldest and the newest regular Python versions on Linux and Windows, as a conda package and, where `pyproject.toml` declares the platform, as a wheel, and tests each package. Jenkins notices the pull request within 3 minutes, waits for that whole run, and then tests and scans every package at once, and benchmarks the newest Linux conda package, sampling each benchmark once. Each cell posts its own status on the pull request, and `intel-ci/pr` sums them up. + +**A release tag.** Actions builds every declared version, and Jenkins tests and scans all of them, and measures the benchmarks properly on the newest Linux conda package, before it publishes the release. A benchmark that fails fails the run; a slower one does not, as nothing compares the timings with a baseline yet. + +To support another Python, add it to `conda-recipe-cf/conda_build_config.yaml` for the conda packages and as a classifier in `pyproject.toml` for the wheels; nothing else lists versions. Try it locally from a conda base environment with conda-build, or with cibuildwheel for wheels: + +```bash +python ci/matrix.py # every platform and Python version +python ci/matrix.py --event pull_request # what a pull request builds +python ci/build.py --python VERSION --output /tmp/out # VERSION: one matrix.py lists +python ci/hooks.py --python VERSION --artifacts /tmp/out --work /tmp/hooks +python ci/build.py --system wheel --python VERSION --output /tmp/wheels +python ci/hooks.py --kind wheel --python VERSION --artifacts /tmp/wheels --work /tmp/wheel-hooks +``` diff --git a/ci/benchmark.py b/ci/benchmark.py new file mode 100644 index 00000000..af04d60b --- /dev/null +++ b/ci/benchmark.py @@ -0,0 +1,90 @@ +""" +Intel Python CI (gh-ci) benchmark of the installed package. + +Intel's Jenkins runs this because the repository has a benchmarks/ directory: +once per pull request or tag, with the conda package of the newest regular +Linux Python installed in a clean conda-forge environment with the benchmark +requirements from ci/config.yml, on a bare-metal benchmarking machine, from an +empty working directory. + +It runs the asv suite in benchmarks/ against the installed package, without a +build of its own, and records the results under the commit being tested, in +$GH_CI_RESULTS, which Jenkins keeps with the build. A pull request samples each +benchmark once, enough to show that every one still runs; a tag measures them +properly. A benchmark that fails fails the cell; a slower one does not, as +nothing compares the timings with a baseline yet. +""" + +import json +import os +import pathlib +import subprocess +import sys + +# asv's own dependency, which reads the comments in its configuration. +import json5 + +ASV_CONFIG = pathlib.Path("benchmarks", "asv.conf.json") + + +def asv_config(source: pathlib.Path, results: pathlib.Path) -> pathlib.Path: + """Writes this run's asv configuration into the working directory. + + It is the repository's own, with absolute paths, since asv reads them from + where the configuration is, and with results under GH_CI_RESULTS. It lists + one branch, HEAD: the checkout is the commit under test, detached, and asv + cannot resolve the repository's branches there. + """ + path = source / ASV_CONFIG + config = json5.loads(path.read_text(encoding="utf-8")) + work = pathlib.Path.cwd() + config.update( + repo=str((path.parent / config.get("repo", "..")).resolve()), + benchmark_dir=str( + (path.parent / config.get("benchmark_dir", "benchmarks")).resolve() + ), + branches=["HEAD"], + results_dir=str(results / "asv"), + html_dir=str(work / "html"), + env_dir=str(work / "env"), + ) + written = work / "asv.conf.json" + written.write_text(json.dumps(config, indent=2), encoding="utf-8") + return written + + +def main() -> int: + """Runs the asv suite against the installed package.""" + config = asv_config( + pathlib.Path(os.environ["GH_CI_SOURCE"]), + pathlib.Path(os.environ["GH_CI_RESULTS"]), + ) + asv = [sys.executable, "-m", "asv"] + # asv keeps its machine description in ~/.asv-machine.json; this one stays + # with the run rather than with the machine's user. + home = pathlib.Path.cwd() / "home" + home.mkdir(exist_ok=True) + env = dict(os.environ, HOME=str(home)) + described = subprocess.run( + asv + ["machine", "--yes", "--config", str(config)], env=env + ) + if described.returncode: + return described.returncode + run = [ + "run", + "--config", + str(config), + f"--environment=existing:{sys.executable}", + f"--set-commit-hash={os.environ['GH_CI_COMMIT']}", + "--no-pull", + "--show-stderr", + "--launch-method=spawn", + ] + if os.environ.get("GH_CI_EVENT") != "tag": + run.append("--quick") + # asv exits 2 when a benchmark failed. + return subprocess.run(asv + run, env=env).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/build.py b/ci/build.py new file mode 100644 index 00000000..fdbb08fc --- /dev/null +++ b/ci/build.py @@ -0,0 +1,343 @@ +""" +Builds this repository's packages for one Python version. + +For the gh-ci GitHub Actions workflow, and for Coverity in Jenkins. + +Copy this file and ci/matrix.py into any repository gh-ci tests. They work out +the build system from the files present, and the Python versions from that +build system's own declaration, so most repositories need no changes to them: + + rattler-build .conda recipe.yaml or recipe/recipe.yaml, versions in + variants.yaml + conda-build .conda conda-recipe-cf/meta.yaml or meta.yaml, versions in + conda_build_config.yaml + cibuildwheel .whl pyproject.toml, versions from its classifiers + +Usage: + python ci/build.py --python 3.14 --output OUT [--system rattler-build] + [--channel conda-forge] + +--python must be one of the declared versions; `python ci/matrix.py` lists them. + +Packages land directly in OUT, ready for actions/upload-artifact. A conda +recipe's own tests run during the build; Jenkins later runs ci/test.py against +the uploaded artifacts, and builds with this script once more under Coverity for +intel-innersource tags. Needs only the Python standard library. rattler-build is +used from PATH, or downloaded at a pinned version for Linux or Windows and +checked against its published sha256; conda-build, or cibuildwheel for wheels, +must already be installed for this interpreter, so run it with the conda base +install's Python to build with conda-build. + +Wheels build with cibuildwheel, in a manylinux container on Linux, as +pyproject.toml's [tool.cibuildwheel] says. Run-time requirements that +pyproject.toml cannot declare, because the conda package gets them from its +recipe instead, such as MKL's libraries, go in [tool.gh-ci] wheel-requires: +build.py adds them to each wheel's metadata. +""" + +import argparse +import base64 +import hashlib +import os +import platform +import shlex +import shutil +import subprocess +import sys +import tempfile +import urllib.request +import zipfile +from pathlib import Path + +import matrix + +ROOT = matrix.ROOT +PACKAGE_SUFFIXES = (".conda", ".tar.bz2", ".whl") +# cibuildwheel's platform part of a build identifier, per system. +WHEEL_PLATFORMS = { + "Linux": "manylinux_x86_64", + "Windows": "win_amd64", + "Darwin": "macosx_x86_64", +} + +# The rattler-build release fetched when none is on PATH, per system, with the +# file name it goes under and the sha256 its release publishes. +RATTLER_BUILD_VERSION = "v0.77.0" +RATTLER_BUILD_RELEASES = { + "Linux": ( + "rattler-build-x86_64-unknown-linux-musl", + "rattler-build", + "5a5e8150c703b21e88a01f9130ea49a7a680198e9ac9414170c0a293fed7e5af", + ), + "Windows": ( + "rattler-build-x86_64-pc-windows-msvc.exe", + "rattler-build.exe", + "e144334f782cee9aa23e26eb8d095c143e1696fbbec7ba7534a156663d4abb2c", + ), +} + + +class BuildError(Exception): + """Raised when the build cannot start or produced nothing.""" + + +def ensure_rattler_build(tools: Path) -> None: + """ + Puts rattler-build on PATH, downloading the pinned release when none is + there. + """ + if shutil.which("rattler-build"): + return + release = RATTLER_BUILD_RELEASES.get(platform.system()) + if release is None or platform.machine().lower() not in ("x86_64", "amd64"): + raise BuildError( + "no rattler-build on PATH, and only the Linux and Windows x86_64 " + "releases are pinned here" + ) + asset, name, sha256 = release + print(f"build: fetching rattler-build {RATTLER_BUILD_VERSION}", flush=True) + url = ( + "https://github.com/prefix-dev/rattler-build/releases/download/" + f"{RATTLER_BUILD_VERSION}/{asset}" + ) + with urllib.request.urlopen(url, timeout=300) as reply: + binary = reply.read() + if hashlib.sha256(binary).hexdigest() != sha256: + raise BuildError( + f"rattler-build {RATTLER_BUILD_VERSION} does not match its pinned " + "sha256" + ) + tools.mkdir(parents=True, exist_ok=True) + target = tools / name + target.write_bytes(binary) + target.chmod(0o755) + os.environ["PATH"] = f'{tools}{os.pathsep}{os.environ.get("PATH", "")}' + + +def cibuildwheel_identifier(python: matrix.Python) -> str: + """The one build cibuildwheel runs, such as cp314t-manylinux_x86_64.""" + tag = WHEEL_PLATFORMS.get(platform.system()) + if tag is None: + raise BuildError( + f"wheels build on Linux, Windows or macOS, not {platform.system()}" + ) + minor, threaded = matrix.version_key(python.version) + return f'cp3{minor}{"t" if threaded else ""}-{tag}' + + +def wheel_requires() -> list[str]: + """[tool.gh-ci] wheel-requires from pyproject.toml.""" + if matrix.TOMLLIB is None: + raise BuildError("reading pyproject.toml needs Python 3.11 or newer") + text = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + tool = matrix.TOMLLIB.loads(text).get("tool", {}).get("gh-ci", {}) + requires = tool.get("wheel-requires", []) + if not isinstance(requires, list) or not all( + isinstance(r, str) for r in requires + ): + raise BuildError("[tool.gh-ci] wheel-requires must be a list of specs") + return requires + + +def record_hash(data: bytes) -> str: + """A file's hash as a wheel's RECORD spells it.""" + digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()) + return "sha256=" + digest.rstrip(b"=").decode("ascii") + + +def add_requirements(wheel: Path, requirements: list[str]) -> None: + """Adds Requires-Dist lines to a built wheel, and updates its RECORD.""" + with zipfile.ZipFile(wheel) as source: + members = [(info, source.read(info)) for info in source.infolist()] + names = [info.filename for info, _ in members] + metadata = next(n for n in names if n.endswith(".dist-info/METADATA")) + record = metadata.removesuffix("METADATA") + "RECORD" + for index, (info, data) in enumerate(members): + if info.filename == metadata: + head, blank, body = data.decode("utf-8").partition("\n\n") + head = head.rstrip("\n") + "".join( + f"\nRequires-Dist: {r}" for r in requirements + ) + text = head + ("\n\n" + body if blank else "\n") + new_metadata = text.encode("utf-8") + members[index] = (info, new_metadata) + for index, (info, data) in enumerate(members): + if info.filename == record: + lines = [ + ( + f"{metadata},{record_hash(new_metadata)}," + f"{len(new_metadata)}" + if line.split(",")[0] == metadata + else line + ) + for line in data.decode("utf-8").splitlines() + ] + members[index] = (info, ("\n".join(lines) + "\n").encode()) + rewritten = wheel.with_suffix(".tmp") + with zipfile.ZipFile(rewritten, "w", zipfile.ZIP_DEFLATED) as target: + for info, data in members: + target.writestr(info, data) + rewritten.replace(wheel) + + +def commands( + system: str, + selector: Path, + python: matrix.Python, + channels: list[str], + scratch: Path, +) -> list[list[str]]: + """ + The commands that build one Python version's packages into scratch. + + Each build system gets the version back exactly as its own file declares it, + and keeps every other variant setting from that file. + """ + channel_args = [arg for channel in channels for arg in ("-c", channel)] + if system == "rattler-build": + return [ + [ + "rattler-build", + "build", + "-r", + str(selector), + "--variant", + f"python={python.spec}", + "--output-dir", + str(scratch / "conda"), + *channel_args, + ] + ] + if system == "conda-build": + # conda runs as a module of this interpreter, so Windows needs no + # conda.bat on PATH. + return [ + [ + sys.executable, + "-m", + "conda", + "build", + str(selector.parent), + "--python", + python.spec, + "--no-anaconda-upload", + "--output-folder", + str(scratch / "conda"), + "--override-channels", + *channel_args, + ] + ] + return [ + [ + sys.executable, + "-m", + "cibuildwheel", + "--only", + cibuildwheel_identifier(python), + "--output-dir", + str(scratch / "wheel"), + str(ROOT), + ] + ] + + +def collect(scratch: Path, output: Path) -> list[Path]: + """Moves built packages into output, flat, and lists them.""" + output.mkdir(parents=True, exist_ok=True) + for package in sorted(scratch.rglob("*")): + if ( + package.is_file() + and package.name.endswith(PACKAGE_SUFFIXES) + and "broken" not in package.parts + ): + shutil.move(str(package), output / package.name) + built = sorted( + path + for path in output.iterdir() + if path.name.endswith(PACKAGE_SUFFIXES) + ) + if not built: + raise BuildError(f"the build left no package in {output}") + return built + + +def main(argv: list[str] | None = None) -> int: + """Builds one Python version.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--python", required=True, help="a declared version, such as 3.14t" + ) + parser.add_argument( + "--output", required=True, type=Path, help="where the packages go" + ) + parser.add_argument( + "--system", + default="", + choices=("", *matrix.SYSTEMS), + help="build system to use instead of the detected one", + ) + parser.add_argument( + "--channel", + action="append", + default=None, + help="conda channel to build against, repeatable; default conda-forge", + ) + default_tools = ( + Path(os.environ.get("RUNNER_TEMP", tempfile.gettempdir())) + / "gh-ci-tools" + ) + parser.add_argument( + "--tools", + type=Path, + default=default_tools, + help="where a downloaded rattler-build goes", + ) + args = parser.parse_args(argv) + + system, selector = matrix.detect(ROOT, args.system) + declared = { + python.version: python for python in matrix.declared(system, selector) + } + if args.python not in declared: + raise BuildError( + f"Python {args.python} is not declared for {system}; " + f'declared: {", ".join(declared)}' + ) + python = declared[args.python] + print( + f"build: {system} from {selector.relative_to(ROOT)} " + f"for Python {python.version} ({python.spec})", + flush=True, + ) + if system == "rattler-build": + ensure_rattler_build(args.tools) + output = args.output.resolve() + with tempfile.TemporaryDirectory() as scratch: + for command in commands( + system, + selector, + python, + args.channel or ["conda-forge"], + Path(scratch), + ): + print(f"+ {shlex.join(command)}", flush=True) + subprocess.run(command, check=True) + built = collect(Path(scratch), output) + if system == "wheel" and (requires := wheel_requires()): + for wheel in built: + add_requirements(wheel, requires) + print("build: added Requires-Dist " + ", ".join(requires)) + print("build: " + ", ".join(path.name for path in built)) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except ( + BuildError, + matrix.DeclarationError, + subprocess.CalledProcessError, + ) as error: + print(f"build: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/config.yml b/ci/config.yml new file mode 100644 index 00000000..a446054d --- /dev/null +++ b/ci/config.yml @@ -0,0 +1,12 @@ +# Intel Python CI (gh-ci) settings for mkl_umath, see ci/README.md +version: 1 +workflow: intel-ci.yml +artifacts: + conda: "mkl_umath {platform} Python {python}" + wheel: "mkl_umath {platform} Wheels Python {python}" +test: + requires: [pytest] +benchmark: + requires: [psutil] + # conda-forge lacks asv for some Python the recipe declares; the abi3 wheel on PyPI fits all. + pip_requires: [asv] diff --git a/ci/hooks.py b/ci/hooks.py new file mode 100644 index 00000000..718b61b1 --- /dev/null +++ b/ci/hooks.py @@ -0,0 +1,440 @@ +""" +Runs ci/test.py against one built artifact, the way Intel Python CI does. + +For the GitHub Actions test job. + +Each Python version's artifact is installed into a fresh conda environment from +a local channel plus conda-forge, pinned to its exact build, with the test +requirements ci/config.yml lists, and the test runs with that environment's +Python from an empty working directory, on Linux or Windows. A wheel's +environment has only Python from conda-forge: the wheel and the test +requirements install with pip, as in the wheel's users' environments. The +interpreter spec and the requirements are the ones Jenkins uses. It gets the +same variables, and nothing else from the runner, as it does in Jenkins, so a +test that passes here passes there. Jenkins runs ci/test.py again on its own +machines, GPUs included; the scans are Jenkins' own and never run here. + +Usage, from any Python 3.10 or newer; it runs itself again with the conda base +install's Python, from $CONDA, which setup-miniconda sets, or the conda on PATH: + python ci/hooks.py --python 3.14 --artifacts DIR --work DIR [--kind wheel] + +Needs conda, conda-package-handling and ruamel.yaml, which every conda base has, +and installs nothing into it. +""" + +import argparse +import hashlib +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +import matrix + +# Dependencies of conda itself; YAML is read with PyYAML, else ruamel.yaml +try: + from conda_package_handling import api as CPH_API +except ModuleNotFoundError: + CPH_API = None +try: + import yaml as PYYAML +except ModuleNotFoundError: + PYYAML = None +try: + from ruamel.yaml import YAML as RUAMEL_YAML +except ModuleNotFoundError: + RUAMEL_YAML = None + +ROOT = matrix.ROOT +IS_WINDOWS = os.name == "nt" +if IS_WINDOWS: + PLATFORM, HOST_SUBDIR = "windows", "win-64" +elif sys.platform == "darwin": + PLATFORM, HOST_SUBDIR = "macos", "osx-64" +else: + PLATFORM, HOST_SUBDIR = "linux", "linux-64" +CONDA_SUFFIXES = (".conda", ".tar.bz2") +# All the test sees of the runner's environment, so no tokens reach it +KEEP = ( + "PATH", + "HOME", + "USER", + "LANG", + "TERM", + "TMPDIR", + "TZ", + "http_proxy", + "https_proxy", + "no_proxy", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "SSL_CERT_FILE", + "REQUESTS_CA_BUNDLE", + "CONDA_PKGS_DIRS", + "SYSTEMROOT", + "SYSTEMDRIVE", + "WINDIR", + "COMSPEC", + "PATHEXT", + "TEMP", + "TMP", + "USERPROFILE", + "APPDATA", + "LOCALAPPDATA", + "PROGRAMDATA", + "NUMBER_OF_PROCESSORS", + "PROCESSOR_ARCHITECTURE", +) + + +class HookError(Exception): + """Raised when the artifact cannot be installed or ci/test.py is missing.""" + + +def run(command: list[str], **options: object) -> None: + """Echoes and runs a command, never through a shell.""" + print(f"+ {shlex.join(command)}", flush=True) + subprocess.run(command, check=True, **options) + + +def conda(*args: str) -> None: + """Runs conda from the base environment running this script.""" + run([sys.executable, "-m", "conda", *args]) + + +def empty_repodata(subdir: str) -> dict: + """The repodata.json of a channel subdir with no packages.""" + return { + "info": {"subdir": subdir}, + "packages": {}, + "packages.conda": {}, + "repodata_version": 1, + } + + +def package_record(package: Path) -> tuple[str, dict]: + """A conda package's subdir and repodata record, from its index.json.""" + if CPH_API is None: + raise HookError( + "run this with a conda base Python, " + "which has conda-package-handling" + ) + with tempfile.TemporaryDirectory() as scratch: + CPH_API.extract(str(package), scratch, components="info") + record = json.loads( + (Path(scratch) / "info" / "index.json").read_text(encoding="utf-8") + ) + data = package.read_bytes() + record.update( + { + "md5": hashlib.md5(data, usedforsecurity=False).hexdigest(), + "sha256": hashlib.sha256(data).hexdigest(), + "size": len(data), + } + ) + subdir = str(record.get("subdir") or "noarch") + if not re.fullmatch(r"[a-z0-9]+(-[a-z0-9_]+)?", subdir): + raise HookError( + f"{package.name} declares an unusable subdir {subdir!r}" + ) + return subdir, record + + +def stage_channel(packages: list[Path], channel: Path) -> None: + """ + Lays the packages out as a local channel, by the subdir each one declares, + and writes its repodata.json the way Jenkins does, without conda-index. + """ + repodata = { + subdir: empty_repodata(subdir) for subdir in ("noarch", HOST_SUBDIR) + } + for package in packages: + subdir, record = package_record(package) + (channel / subdir).mkdir(parents=True, exist_ok=True) + shutil.copy2(package, channel / subdir / package.name) + section = ( + "packages.conda" if package.name.endswith(".conda") else "packages" + ) + repodata.setdefault(subdir, empty_repodata(subdir))[section][ + package.name + ] = record + for subdir, data in repodata.items(): + (channel / subdir).mkdir(parents=True, exist_ok=True) + (channel / subdir / "repodata.json").write_text( + json.dumps(data, indent=2) + "\n", encoding="utf-8" + ) + + +def test_python(version: str) -> str: + """The interpreter spec for the test environment, as Jenkins spells it.""" + minor, threaded = matrix.version_key(version) + # The .* matters: conda reads a bare 3.11 as exactly 3.11.0 + if minor < matrix.FREE_THREADED_MINOR: + return f"3.{minor}.*" + # conda-forge marks the ABI from 3.13, so 3.14 can't resolve to 3.14t + return f'3.{minor}.* *_cp3{minor}{"t" if threaded else ""}' + + +def test_requirements() -> tuple[list[str], list[str], list[str] | None]: + """ + The conda-forge and pip packages ci/config.yml installs next to the + artifact, and a wheel's PyPI requirements, None when it gives none. + """ + text = (ROOT / "ci" / "config.yml").read_text(encoding="utf-8") + if PYYAML is not None: + config = PYYAML.safe_load(text) + elif RUAMEL_YAML is not None: + config = RUAMEL_YAML(typ="safe", pure=True).load(text) + else: + raise HookError("reading ci/config.yml needs PyYAML or ruamel.yaml") + test = (config or {}).get("test") or {} + lists = [] + for key in ("requires", "pip_requires", "wheel_requires"): + values = test.get(key) + if values is not None and ( + not isinstance(values, list) + or not all(isinstance(v, str) for v in values) + ): + raise HookError( + f"ci/config.yml: test.{key} must be a list of package specs" + ) + lists.append(values) + return lists[0] or [], lists[1] or [], lists[2] + + +def exact_spec(package: Path) -> str: + """name=version=build from a conda package file name.""" + stem = package.name + for suffix in CONDA_SUFFIXES: + stem = stem.removesuffix(suffix) + name, version, build = stem.rsplit("-", 2) + return f"{name}={version}={build}" + + +def env_python(env_dir: Path) -> Path: + """The interpreter of a conda environment.""" + return env_dir / "python.exe" if IS_WINDOWS else env_dir / "bin" / "python" + + +def env_path(env_dir: Path) -> list[Path]: + """ + The directories conda's activation would put first on PATH for an + environment. + """ + if not IS_WINDOWS: + return [env_dir / "bin"] + library = env_dir / "Library" + return [ + env_dir, + library / "mingw-w64" / "bin", + library / "usr" / "bin", + library / "bin", + env_dir / "Scripts", + env_dir / "bin", + ] + + +def test_env( + env_dir: Path, artifacts: Path, results: Path, args: argparse.Namespace +) -> dict: + """The variables Jenkins gives ci/test.py, and no runner tokens.""" + env = {key: value for key, value in os.environ.items() if key in KEEP} + env["PATH"] = os.pathsep.join( + [*map(str, env_path(env_dir)), env.get("PATH", "")] + ) + env.update( + { + "CONDA_PREFIX": str(env_dir), + "GH_CI_ARTIFACTS": str(artifacts), + "GH_CI_SOURCE": str(ROOT), + "GH_CI_RESULTS": str(results), + "GH_CI_KIND": args.kind, + "GH_CI_PYTHON": args.python, + "GH_CI_PLATFORM": PLATFORM, + "GH_CI_EVENT": args.event, + "GH_CI_COMMIT": args.commit, + } + ) + return env + + +def summarize(lines: list[str]) -> None: + """Adds the results to the job summary page, in GitHub Actions.""" + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as page: + page.write("\n".join(lines) + "\n") + + +def install(args: argparse.Namespace) -> Path: + """ + Creates the test environment with the artifact, pinned to its exact build; + returns it. + """ + system, selector = matrix.detect( + ROOT, "wheel" if args.kind == "wheel" else "" + ) + declared = { + python.version: python for python in matrix.declared(system, selector) + } + if args.python not in declared: + raise HookError( + f"Python {args.python} is not declared; " + f'declared: {", ".join(declared)}' + ) + suffixes = (".whl",) if args.kind == "wheel" else CONDA_SUFFIXES + packages = sorted( + path + for path in args.artifacts.rglob("*") + if path.name.endswith(suffixes) + ) + if not packages: + raise HookError(f"{args.artifacts} holds no {args.kind} package") + requires, pip_requires, wheel_requires = test_requirements() + env_dir = args.work / "env" + pip = [str(env_python(env_dir)), "-m", "pip", "install", "--no-input"] + python = f"python {test_python(args.python)}" + if args.kind == "wheel": + conda_create(env_dir, python, "pip") + wheel_specs = requires if wheel_requires is None else wheel_requires + run([*pip, *map(str, packages), *wheel_specs]) + else: + channel = args.work / "channel" + stage_channel(packages, channel) + conda_create( + env_dir, + python, + *requires, + *map(exact_spec, packages), + channels=(channel.as_uri(),), + ) + if pip_requires: + run([*pip, *pip_requires]) + return env_dir + + +def conda_create( + env_dir: Path, *specs: str, channels: tuple[str, ...] = () +) -> None: + """Creates a conda environment from conda-forge and channels.""" + conda( + "create", + "-y", + "-q", + "-p", + str(env_dir), + "--override-channels", + *(arg for channel in channels for arg in ("-c", channel)), + "-c", + "conda-forge", + *specs, + ) + + +def run_test(env_dir: Path, args: argparse.Namespace) -> int: + """Runs ci/test.py with the environment's Python; returns its status.""" + script = ROOT / "ci" / "test.py" + if not script.is_file(): + raise HookError("ci/test.py is required and missing") + results, scratch = ( + args.work / "results" / "test", + args.work / "scratch" / "test", + ) + for directory in (results, scratch): + directory.mkdir(parents=True, exist_ok=True) + command = [str(env_python(env_dir)), str(script)] + print(f"+ {shlex.join(command)}", flush=True) + status = subprocess.run( + command, + cwd=scratch, + check=False, + env=test_env(env_dir, args.artifacts, results, args), + ).returncode + result = "passed" if status == 0 else f"failed, exit {status}" + summarize( + [ + f"### {matrix.PLATFORMS[PLATFORM]} Python {args.python}" + f" {args.kind}", + "", + "| Test | Result |", + "|---|---|", + f"| ci/test.py | {result} |", + ] + ) + return status + + +def conda_base_python() -> Path: + """The conda base install's Python: from $CONDA, else the conda on PATH.""" + root = os.environ.get("CONDA", "") + if not root: + found = shutil.which("conda") + if not found: + raise HookError( + "no conda base install: set CONDA or put conda on PATH" + ) + # conda lives in bin, Scripts or condabin, one level below the install. + root = str(Path(found).resolve().parent.parent) + python = env_python(Path(root)) + if not python.is_file(): + raise HookError( + f"{root} is not a conda base install; it has no {python.name}" + ) + return python + + +def main(argv: list[str] | None = None) -> int: + """Installs the artifact, then runs ci/test.py against it.""" + if CPH_API is None: + # Re-run with base's Python, which has conda-package-handling + command = [ + str(conda_base_python()), + str(Path(__file__).resolve()), + *(sys.argv[1:] if argv is None else argv), + ] + return subprocess.run(command, check=False).returncode + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--python", required=True, help="a declared version, such as 3.14t" + ) + parser.add_argument( + "--artifacts", required=True, type=Path, help="the downloaded artifact" + ) + parser.add_argument( + "--work", required=True, type=Path, help="an empty scratch directory" + ) + parser.add_argument( + "--event", default="push", choices=("pr", "tag", "push") + ) + parser.add_argument("--commit", default="") + parser.add_argument( + "--kind", + default="conda", + choices=("conda", "wheel"), + help="what the artifact holds", + ) + args = parser.parse_args(argv) + args.artifacts, args.work = args.artifacts.resolve(), args.work.resolve() + if args.work.exists() and ( + not args.work.is_dir() or any(args.work.iterdir()) + ): + raise HookError(f"--work {args.work} must be an empty directory") + return run_test(install(args), args) + + +if __name__ == "__main__": + try: + sys.exit(main()) + except ( + HookError, + matrix.DeclarationError, + subprocess.CalledProcessError, + ) as error: + print(f"hooks: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/install_icx.py b/ci/install_icx.py new file mode 100644 index 00000000..631eafbc --- /dev/null +++ b/ci/install_icx.py @@ -0,0 +1,62 @@ +""" +Installs Intel's icx for mkl_umath's Windows wheel builds. + +cibuildwheel runs this before the builds, as pyproject.toml says. It fetches +micromamba, checked against its published sha256, and creates PREFIX with +conda-forge's dpcpp_win-64, the compiler the conda recipe builds with. + +Usage: + python ci/install_icx.py PREFIX +""" + +import hashlib +import subprocess +import sys +import tempfile +import urllib.request +from pathlib import Path + +MICROMAMBA = ( + "https://github.com/mamba-org/micromamba-releases/releases/download/" + "2.10.0-0/micromamba-win-64" +) +MICROMAMBA_SHA256 = ( + "f8b4284bd4eb1134122eaf93b3ea6c7be567e596de5829e1a6f478e34e41974a" +) +COMPILER = "dpcpp_win-64=2026.2.0" + + +def main() -> int: + """Creates PREFIX with the compiler.""" + prefix = Path(sys.argv[1]) + with tempfile.TemporaryDirectory() as scratch: + micromamba = Path(scratch) / "micromamba.exe" + with urllib.request.urlopen(MICROMAMBA, timeout=300) as response: + micromamba.write_bytes(response.read()) + digest = hashlib.sha256(micromamba.read_bytes()).hexdigest() + if digest != MICROMAMBA_SHA256: + print( + f"install_icx: micromamba has sha256 {digest}, " + f"not {MICROMAMBA_SHA256}", + file=sys.stderr, + ) + return 1 + command = [ + str(micromamba), + "create", + "-y", + "-r", + str(Path(scratch) / "root"), + "-p", + str(prefix), + "--override-channels", + "-c", + "conda-forge", + COMPILER, + ] + print("+ " + " ".join(command), flush=True) + return subprocess.run(command, check=False).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/matrix.py b/ci/matrix.py new file mode 100644 index 00000000..7a44d341 --- /dev/null +++ b/ci/matrix.py @@ -0,0 +1,653 @@ +""" +The platforms and Python versions this repository builds. + +Both are read from the repository's own build files. + +Every build system already has a standard place for both, so each is written +once, where the build reads it, and the GitHub Actions matrix and Intel Python +CI (gh-ci) both follow it: + + rattler-build, selected by recipe.yaml + Python versions python: in variants.yaml next to the recipe + platforms build: skip: in the recipe + conda-build, selected by meta.yaml + Python versions python: in conda_build_config.yaml next to the recipe + platforms skip: true # [selector] lines in the recipe + wheel, selected by pyproject.toml + Python versions the Programming Language :: Python :: 3.N classifiers; + any Free Threading classifier adds 3.Nt for each from 3.14 + platforms the Operating System :: classifiers + +A recipe builds on every platform gh-ci builds, Linux, Windows and macOS on +Intel, unless a skip that depends on the platform alone rules one out, such as +`skip: [win]` or `skip: true # [osx]`; a skip that also depends on something +else, such as `win and py < 311` or `osx and arm64`, skips some builds, not the +platform. A pyproject.toml without Operating System classifiers, or with OS +Independent, builds on all three. Of the declared platforms, the matrix holds +those GitHub Actions has runners for, BUILT_PLATFORMS. macOS builds and tests in +GitHub Actions only: Jenkins tests, scans and releases Linux and Windows. + +meson-python, scikit-build-core and every other PEP 517 backend declare these in +pyproject.toml too, so the wheel row covers them. Another convention, such as +pixi.toml, is one more reader in READERS and PLATFORM_READERS and, if it is a +new build system, one more entry in SYSTEMS. + +Usage: + python ci/matrix.py [--system SYSTEM] [--event EVENT] [--github-output NAME] + +Prints the build matrix as JSON for strategy.matrix: {"include": [{"platform": +"linux", "name": "Linux", "python": "3.14"}, ...]}, every platform with every +Python version. For a pull_request event only the oldest and the newest regular +versions are listed, which is all gh-ci tests on a pull request. --github-output +also writes NAME= to $GITHUB_OUTPUT, for later jobs. Standard library +only; reading pyproject.toml needs Python 3.11 or newer for tomllib. +""" + +import argparse +import ast +import json +import os +import re +import sys +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +# Only the wheel reader needs it, and only it needs Python 3.11 or newer. +try: + import tomllib as TOMLLIB +except ModuleNotFoundError: + TOMLLIB = None + +ROOT = Path(__file__).resolve().parent.parent + +# Build system -> files that select it, in the order they are looked for. The +# conda-forge recipe comes first: gh-ci tests from conda-forge alone, and +# conda-recipe/ builds against Intel's channel. +SYSTEMS = { + "rattler-build": ("recipe.yaml", "recipe/recipe.yaml"), + "conda-build": ( + "conda-recipe-cf/meta.yaml", + "meta.yaml", + "conda-recipe/meta.yaml", + "recipe/meta.yaml", + ), + "wheel": ("pyproject.toml",), +} + +# The platforms gh-ci builds and tests, and how artifact names spell them. +PLATFORMS = {"linux": "Linux", "windows": "Windows", "macos": "macOS"} +# The platforms the workflow can build on: GitHub-hosted runners have all +# three. A declared platform missing here is reported and left out. +BUILT_PLATFORMS = ("linux", "windows", "macos") +# What a platform-only selector means on each platform, for both conda-build and +# rattler-build. +SELECTORS = { + "linux": { + "linux": True, + "linux64": True, + "unix": True, + "win": False, + "win64": False, + "win32": False, + "osx": False, + "osx64": False, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, + "windows": { + "linux": False, + "linux64": False, + "unix": False, + "win": True, + "win64": True, + "win32": False, + "osx": False, + "osx64": False, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, + # Intel Macs only, as MKL has no Apple silicon build. + "macos": { + "linux": False, + "linux64": False, + "unix": True, + "win": False, + "win64": False, + "win32": False, + "osx": True, + "osx64": True, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, +} +CONDA_SKIP_RE = re.compile(r"\s*skip\s*:\s*(?:true|True|yes)\s*#\s*\[(.+)\]\s*") +OS_CLASSIFIERS = { + "Operating System :: OS Independent": tuple(PLATFORMS), + "Operating System :: POSIX": ("linux",), + "Operating System :: Unix": ("linux",), + "Operating System :: POSIX :: Linux": ("linux",), + "Operating System :: Microsoft :: Windows": ("windows",), + "Operating System :: MacOS": ("macos",), +} + +PULL_REQUEST_EVENTS = ("pull_request",) +OLDEST_MINOR = 10 +FREE_THREADED_MINOR = 13 +# A Free Threading classifier adds 3.Nt from 3.14, the first supported one, +# which is also the oldest cibuildwheel builds. +CLASSIFIED_FREE_THREADED_MINOR = 14 + +# A conda python variant: 3.12, 3.12.*, 3.12.* *_cpython, 3.13.* *_cp313 or +# 3.14.* *_cp314t. +CONDA_SPEC_RE = re.compile( + r"3\.(\d+)(?:\.\*)?(?:\s+\*_(?:cpython|cp3(\d+)(t?)))?" +) +CLASSIFIER_RE = re.compile(r"Programming Language :: Python :: 3\.(\d+)") +FREE_THREADING_CLASSIFIER = "Programming Language :: Python :: Free Threading" +LOWER_BOUND_RE = re.compile(r">=\s*3\.(\d+)") + + +class DeclarationError(Exception): + """ + Raised when the build files declare no Python versions, or ones this cannot + read. + """ + + +@dataclass(frozen=True) +class Python: + """One declared Python version.""" + + version: str # as artifact names and gh-ci spell it: 3.14 or 3.14t + spec: str # as the build system spells it, passed back to it unchanged + + +def version_key(version: str) -> tuple[int, bool]: + """(minor, free-threaded); also the sort order.""" + return int(version.rstrip("t").split(".")[1]), version.endswith("t") + + +def checked(minor: int, threaded: bool, where: str) -> str: + """The version string, once it is one gh-ci can test.""" + if minor < OLDEST_MINOR or (threaded and minor < FREE_THREADED_MINOR): + raise DeclarationError( + f'{where}: 3.{minor}{"t" if threaded else ""} is not supported; ' + f"the oldest is 3.{OLDEST_MINOR}, free-threaded from " + f"3.{FREE_THREADED_MINOR}" + ) + return f'3.{minor}{"t" if threaded else ""}' + + +def _strip_comment(line: str, where: str) -> str: + """ + The line without a trailing comment; conda-build selectors are refused, not + guessed at. + """ + in_quote = "" + for index, char in enumerate(line): + if char in "\"'" and in_quote in ("", char): + in_quote = "" if in_quote else char + elif ( + char == "#" + and not in_quote + and (index == 0 or line[index - 1].isspace()) + ): + if line[index:].lstrip("# ").startswith("["): + raise DeclarationError( + f"{where}: selectors in the python list are not supported" + ) + return line[:index].rstrip() + return line.rstrip() + + +def _unquote(value: str, where: str) -> str: + """A plain or quoted YAML scalar.""" + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + return value[1:-1] + if not value or value[0] in "{[&*!|>" or ":" in value: + raise DeclarationError( + f"{where}: write each python entry as a plain value, not {value!r}" + ) + return value + + +def yaml_python_list(text: str, where: str) -> list[str]: + """ + The top-level python: list of a variants file, without a YAML library. + + Accepts a block list or a one-line [a, b] list of plain or quoted values. + Anything richer, such as if/then selectors or templating, is refused so the + matrix is never guessed. + """ + lines = text.splitlines() + for number, line in enumerate(lines): + if not re.match(r"python\s*:", line): + continue + rest = _strip_comment(line.split(":", 1)[1], where).strip() + if rest.startswith("[") and rest.endswith("]"): + return [ + _unquote(item, where) + for item in rest[1:-1].split(",") + if item.strip() + ] + if rest: + raise DeclarationError(f"{where}: python must be a list") + values = [] + for item in lines[number + 1 :]: + # Checked before the comment, as the next key may have a selector + if item[:1] not in ("", " ", "\t", "-", "#"): + break # the next top-level key + entry = _strip_comment(item, where).lstrip() + if not entry: + continue + if not entry.startswith("- "): + raise DeclarationError( + f"{where}: python must be a plain list of versions" + ) + values.append(_unquote(entry[2:], where)) + return values + raise DeclarationError(f"{where} has no top-level python: list") + + +def from_conda_variants(path: Path) -> list[Python]: + """The python: list of a variants.yaml or a conda_build_config.yaml.""" + where = ( + path.relative_to(ROOT).as_posix() + if path.is_relative_to(ROOT) + else str(path) + ) + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError as ex: + raise DeclarationError( + f"{where} is missing; list the Python versions to build there" + ) from ex + pythons = [] + for spec in yaml_python_list(text, where): + match = CONDA_SPEC_RE.fullmatch(spec) + if not match or (match[2] and match[2] != match[1]): + raise DeclarationError( + f"{where}: {spec!r} is not a CPython variant such as " + '"3.13.* *_cp313" or "3.14.* *_cp314t"' + ) + pythons.append( + Python(checked(int(match[1]), bool(match[3]), where), spec) + ) + return pythons + + +def from_pyproject(path: Path) -> list[Python]: + """ + The 3.N classifiers in pyproject.toml, plus 3.Nt from 3.14 if free-threading + is declared. + """ + if TOMLLIB is None: + raise DeclarationError( + "reading pyproject.toml needs Python 3.11 or newer" + ) + project = TOMLLIB.loads(path.read_text(encoding="utf-8")).get("project", {}) + classifiers = project.get("classifiers", []) + minors = sorted( + { + int(match[1]) + for classifier in classifiers + if (match := CLASSIFIER_RE.fullmatch(classifier)) + } + ) + if not minors: + raise DeclarationError( + "pyproject.toml lists no Programming Language :: Python :: 3.N " + "classifiers" + ) + bound = LOWER_BOUND_RE.search(project.get("requires-python", "")) + if bound and minors[0] < int(bound[1]): + raise DeclarationError( + f"pyproject.toml classifies 3.{minors[0]}, below requires-python " + f'{project["requires-python"]}' + ) + threaded = any( + classifier.startswith(FREE_THREADING_CLASSIFIER) + for classifier in classifiers + ) + versions = [checked(minor, False, "pyproject.toml") for minor in minors] + versions += [ + f"3.{minor}t" + for minor in minors + if threaded and minor >= CLASSIFIED_FREE_THREADED_MINOR + ] + return [Python(version, version) for version in versions] + + +def skips_platform(expression: str, platform: str) -> bool | None: + """ + Whether a skip expression rules out a whole platform: True or False when it + depends on the platform alone, None when it also depends on anything else, + such as the Python version. + """ + try: + tree = ast.parse( + expression.strip().removeprefix("${{").removesuffix("}}").strip(), + mode="eval", + ).body + except SyntaxError: + return None + + def value(node: ast.AST) -> bool: + if isinstance(node, ast.BoolOp): + values = [value(item) for item in node.values] + return all(values) if isinstance(node.op, ast.And) else any(values) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + return not value(node.operand) + if isinstance(node, ast.Name) and node.id in SELECTORS[platform]: + return SELECTORS[platform][node.id] + if isinstance(node, ast.Constant) and isinstance(node.value, bool): + return node.value + raise ValueError(ast.dump(node)) + + try: + return value(tree) + except ValueError: + return None + + +def platforms_without(skips: list[str], where: str) -> list[str]: + """Every platform gh-ci builds, but those a platform-only skip rules out.""" + kept = [ + platform + for platform in PLATFORMS + if not any(skips_platform(skip, platform) for skip in skips) + ] + if not kept: + raise DeclarationError( + f"{where} skips every platform gh-ci builds: " + + ", ".join(PLATFORMS.values()) + ) + return kept + + +def yaml_build_skips(text: str, where: str) -> list[str]: + """ + The skip: entries directly under a rattler-build recipe's top-level build:, + as a list of plain or quoted values, a one-line [a, b] list, or one value; + none when there is no skip. + """ + lines = text.splitlines() + starts = [ + number + for number, line in enumerate(lines) + if re.match(r"build\s*:\s*(#.*)?$", line) + ] + if not starts: + return [] + child = "" + for number in range(starts[0] + 1, len(lines)): + line = ( + _strip_comment(lines[number], where) + if "#" in lines[number] + else lines[number] + ) + if not line.strip(): + continue + if not line[0].isspace(): + return [] # the next top-level key + indent = line[: len(line) - len(line.lstrip())] + child = child or indent + match = ( + re.match(r"skip\s*:(.*)", line.lstrip()) + if indent == child + else None + ) + if not match: + continue + rest = match[1].strip() + if rest.startswith("[") and rest.endswith("]"): + return [ + _unquote(item, where) + for item in rest[1:-1].split(",") + if item.strip() + ] + if rest: + return [_unquote(rest, where)] + values = [] + for item in lines[number + 1 :]: + entry = item.strip() + if not entry or entry.startswith("#"): + continue + if not entry.startswith("- "): + break + values.append(_unquote(entry[2:].split(" #")[0], where)) + return values + return [] + + +def from_rattler_recipe(recipe: Path) -> list[str]: + """The platforms a rattler-build recipe builds on, from its build: skip:.""" + where = ( + recipe.relative_to(ROOT).as_posix() + if recipe.is_relative_to(ROOT) + else str(recipe) + ) + return platforms_without( + yaml_build_skips(recipe.read_text(encoding="utf-8"), where), where + ) + + +def from_conda_recipe(recipe: Path) -> list[str]: + """ + The platforms a conda-build meta.yaml builds on, from its skip: true # + [selector] lines. + """ + where = ( + recipe.relative_to(ROOT).as_posix() + if recipe.is_relative_to(ROOT) + else str(recipe) + ) + skips = [ + match[1] + for line in recipe.read_text(encoding="utf-8").splitlines() + if (match := CONDA_SKIP_RE.fullmatch(line)) + ] + return platforms_without(skips, where) + + +def from_os_classifiers(path: Path) -> list[str]: + """ + The platforms pyproject.toml's Operating System classifiers name; all of + them when it names none. + """ + if TOMLLIB is None: + raise DeclarationError( + "reading pyproject.toml needs Python 3.11 or newer" + ) + classifiers = ( + TOMLLIB.loads(path.read_text(encoding="utf-8")) + .get("project", {}) + .get("classifiers", []) + ) + named = [ + classifier + for classifier in classifiers + if classifier.startswith("Operating System ::") + ] + if not named: + return list(PLATFORMS) + found = { + platform + for classifier in named + for prefix, platforms in OS_CLASSIFIERS.items() + if classifier == prefix or classifier.startswith(prefix + " ::") + for platform in platforms + } + if not found: + raise DeclarationError( + "pyproject.toml names no Operating System gh-ci builds: " + + ", ".join(PLATFORMS.values()) + ) + return [platform for platform in PLATFORMS if platform in found] + + +# Build system -> where it declares Python versions, given the file that +# selected it. +READERS: dict[str, Callable[[Path], list[Python]]] = { + "rattler-build": lambda recipe: from_conda_variants( + recipe.parent / "variants.yaml" + ), + "conda-build": lambda recipe: from_conda_variants( + recipe.parent / "conda_build_config.yaml" + ), + "wheel": from_pyproject, +} + +# Build system -> where it declares the platforms it builds on, given the file +# that selected it. +PLATFORM_READERS: dict[str, Callable[[Path], list[str]]] = { + "rattler-build": from_rattler_recipe, + "conda-build": from_conda_recipe, + "wheel": from_os_classifiers, +} + + +def detect(root: Path, system: str = "") -> tuple[str, Path]: + """The build system and the file that selects it.""" + for name, candidates in SYSTEMS.items(): + if system and name != system: + continue + for candidate in candidates: + if (root / candidate).is_file(): + return name, root / candidate + wanted = system or " or ".join(SYSTEMS) + raise DeclarationError( + f"no {wanted} input found; looked for " + + ", ".join(c for name, cs in SYSTEMS.items() for c in cs) + ) + + +def declared(system: str, selector: Path) -> list[Python]: + """The build system's Python versions, sorted, each listed once.""" + pythons = READERS[system](selector) + versions = [python.version for python in pythons] + if not pythons: + raise DeclarationError(f"{system} declares no Python versions") + if len(set(versions)) != len(versions): + raise DeclarationError( + f"{system} declares a Python version twice: {versions}" + ) + return sorted(pythons, key=lambda python: version_key(python.version)) + + +def declared_platforms(system: str, selector: Path) -> list[str]: + """The platforms the build system builds on, in PLATFORMS order.""" + return PLATFORM_READERS[system](selector) + + +def for_event(pythons: list[Python], event: str) -> list[Python]: + """All of them, or for a pull request the oldest and newest regular ones.""" + if event not in PULL_REQUEST_EVENTS: + return pythons + regular = [python for python in pythons if not python.version.endswith("t")] + regular = regular or pythons + return list(dict.fromkeys(regular[:1] + regular[-1:])) + + +def build_matrix(system: str, selector: Path, event: str) -> dict: + """ + Every declared platform Actions can build on, with every Python version the + event builds. + """ + pythons = for_event(declared(system, selector), event) + platforms = declared_platforms(system, selector) + waiting = [ + PLATFORMS[platform] + for platform in platforms + if platform not in BUILT_PLATFORMS + ] + if waiting: + print( + f'matrix: {", ".join(waiting)} declared but not built: ' + "no runners for it yet", + file=sys.stderr, + ) + built = [platform for platform in platforms if platform in BUILT_PLATFORMS] + if not built: + raise DeclarationError( + "the build files declare only platforms there are no runners " + "for yet: " + ", ".join(waiting) + ) + return { + "include": [ + { + "platform": platform, + "name": PLATFORMS[platform], + "python": python.version, + } + for platform in built + for python in pythons + ] + } + + +def main(argv: list[str] | None = None) -> int: + """Prints the matrix to build; writes it for GitHub Actions if asked.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--system", + default="", + choices=("", *SYSTEMS), + help="build system to read instead of the detected one", + ) + parser.add_argument( + "--event", default="", help="GitHub event name, such as pull_request" + ) + parser.add_argument( + "--github-output", + default="", + metavar="NAME", + help="also write NAME= to $GITHUB_OUTPUT", + ) + args = parser.parse_args(argv) + if args.github_output: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_-]*", args.github_output): + raise DeclarationError( + f"{args.github_output!r} is not an output name" + ) + if not os.environ.get("GITHUB_OUTPUT"): + raise DeclarationError( + "--github-output needs $GITHUB_OUTPUT, which Actions sets" + ) + system, selector = detect(ROOT, args.system) + matrix = build_matrix(system, selector, args.event) + cells = ", ".join( + f'{cell["name"]} {cell["python"]}' for cell in matrix["include"] + ) + print( + f"matrix: {system} from {selector.relative_to(ROOT)}: {cells}", + file=sys.stderr, + ) + text = json.dumps(matrix) + print(text) + if args.github_output: + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"{args.github_output}={text}\n") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except DeclarationError as error: + print(f"matrix: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/test.py b/ci/test.py new file mode 100644 index 00000000..4d4e998c --- /dev/null +++ b/ci/test.py @@ -0,0 +1,53 @@ +""" +Intel Python CI (gh-ci) test for mkl_umath. + +GitHub Actions, through ci/hooks.py, and Intel's Jenkins both run this with one +built conda package installed in a clean conda-forge environment, from an empty +working directory, so the tests import the installed mkl_umath rather than the +source tree. The test requirements come from ci/config.yml. Writes a JUnit +report for Jenkins. +""" + +import os +import pathlib +import sys +import sysconfig + +import pytest + +PACKAGE = "mkl_umath" + + +def running_python() -> str: + """This interpreter's version as gh-ci spells it, such as 3.14t.""" + version = f"{sys.version_info.major}.{sys.version_info.minor}" + threaded = sysconfig.get_config_var("Py_GIL_DISABLED") + return version + ("t" if threaded else "") + + +def main() -> int: + """Runs the installed package's test suite.""" + expected = os.environ["GH_CI_PYTHON"] + if running_python() != expected: + print(f"running Python {running_python()}, the cell is {expected}") + return 1 + # Every cell of a Jenkins build reports into the same build, so each names + # its own suite. + cell = f"{os.environ.get('GH_CI_PLATFORM', 'linux')}.py{expected}" + report = pathlib.Path(os.environ["GH_CI_RESULTS"]) / f"{PACKAGE}.xml" + return pytest.main( + [ + "-v", + "-p", + "no:cacheprovider", + f"--junitxml={report}", + "-o", + f"junit_suite_name={PACKAGE} {cell}", + "--pyargs", + PACKAGE, + ] + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/conda-recipe-cf/AGENTS.md b/conda-recipe-cf/AGENTS.md index dcc7eef5..8e0ffcd0 100644 --- a/conda-recipe-cf/AGENTS.md +++ b/conda-recipe-cf/AGENTS.md @@ -1,21 +1,13 @@ # AGENTS.md — conda-recipe-cf/ -Conda-forge compatible build recipe (alternative to Intel channel recipe). - -## Difference from conda-recipe/ -- No `conda_build_config.yaml` (uses conda-forge defaults) -- May use different compiler toolchain -- For conda-forge feedstock integration (if upstreamed) +The conda recipe, built against conda-forge only. ## Files -- **meta.yaml** — conda-forge compatible metadata, test commands (`pytest --pyargs mkl_umath`) +- **meta.yaml** — metadata, dependencies, test commands (`pytest --pyargs mkl_umath`) - **build.sh** / **bld.bat** — platform build scripts +- **conda_build_config.yaml** — the Python versions Intel Python CI builds, NumPy 2 and the compilers; + its `python:` list is read by `ci/matrix.py`: one version per line, no selectors -## Status -- Not currently used in main CI workflows -- Maintained for potential conda-forge submission -- Use `conda-recipe/` for Intel channel builds - -## Notes -- If upstreaming to conda-forge, this recipe should be preferred -- Compiler requirements may differ (Clang/GCC vs Intel icx) +## CI +- Built and tested by `conda-package-cf.yml` and by Intel Python CI (`intel-ci.yml`) +- Wheels don't come from this recipe: `intel-ci.yml` builds them from `pyproject.toml` with `ci/build.py` diff --git a/conda-recipe-cf/conda_build_config.yaml b/conda-recipe-cf/conda_build_config.yaml index 6e19b3ef..74eeb53b 100644 --- a/conda-recipe-cf/conda_build_config.yaml +++ b/conda-recipe-cf/conda_build_config.yaml @@ -1,3 +1,12 @@ +# Python versions for ci/matrix.py, one per line, without selectors +python: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14.* *_cp314" + - "3.14.* *_cp314t" + - "3.15.* *_cp315" numpy: # as in conda-forge's global pinning; CI overrides it with --numpy - '2' c_compiler: # [linux] diff --git a/conda-recipe-cf/meta.yaml b/conda-recipe-cf/meta.yaml index 60d7c8da..ef2d494a 100644 --- a/conda-recipe-cf/meta.yaml +++ b/conda-recipe-cf/meta.yaml @@ -10,6 +10,7 @@ source: build: number: {{ buildnumber }} + skip: true # [osx] ignore_run_exports_from: - {{ compiler('dpcpp') }} # [linux] ignore_run_exports: @@ -58,3 +59,12 @@ about: license: BSD-3-Clause license_file: LICENSE.txt summary: Universal functions for real and complex floating point arrays powered by Intel(R) Math Kernel Library Vector (Intel(R) MKL) and Intel(R) Short Vector Math Library (Intel(R) SVML) + description: | + LEGAL NOTICE: Use of this software package is subject to the + software license agreement (as set forth above, in the license section of + the installed Conda package and/or the README file) and all notices, + disclaimers or license terms for third party or open source software + included in or with the software. +

+ EULA: BSD-3-Clause +

diff --git a/conda-recipe/AGENTS.md b/conda-recipe/AGENTS.md deleted file mode 100644 index e378ac6d..00000000 --- a/conda-recipe/AGENTS.md +++ /dev/null @@ -1,30 +0,0 @@ -# AGENTS.md — conda-recipe/ - -Conda package build recipe for Intel channel distribution. - -## Files -- **meta.yaml** — package metadata, dependencies, build requirements, test commands (`pytest --pyargs mkl_umath`) -- **build.sh** — Linux build script -- **bld.bat** — Windows build script -- **conda_build_config.yaml** — build matrix (Python versions, numpy pins) - -## Build configuration -- **Channels:** `https://software.repos.intel.com/python/conda`, `conda-forge` -- **Python versions:** 3.10, 3.11, 3.12, 3.13, 3.14, 3.15 -- **Build system:** meson-python (via `python -m build`) -- **Compilers:** Intel C compiler (icx) -- **Dependencies:** mkl-devel, intel-openmp, dpcpp_{linux,win}-64, numpy - -## Build outputs -- Conda package: `mkl_umath--.conda` -- Platform-specific: `linux-64/`, `win-64/` - -## CI usage -- Built in `.github/workflows/conda-package.yml` -- Artifacts uploaded per Python version -- Test stage uses built artifacts from channel - -## Maintenance -- Keep `conda_build_config.yaml` in sync with CI matrix -- NumPy pin: must match Intel channel NumPy versions -- MKL version: track oneAPI releases diff --git a/conda-recipe/bld.bat b/conda-recipe/bld.bat deleted file mode 100644 index 58178a84..00000000 --- a/conda-recipe/bld.bat +++ /dev/null @@ -1,25 +0,0 @@ -REM A workaround for activate-dpcpp.bat issue to be addressed in 2021.4 -set "LIB=%BUILD_PREFIX%\Library\lib;%BUILD_PREFIX%\compiler\lib;%LIB%" -set "INCLUDE=%BUILD_PREFIX%\include;%INCLUDE%" - -set "CC=icx" -set "CXX=icx" - -%PYTHON% -m build -w -n -x -if %ERRORLEVEL% neq 0 exit 1 - -for /f %%f in ('dir /b /S .\dist') do ( - %PYTHON% -m pip install %%f ^ - --no-build-isolation ^ - --no-deps ^ - --only-binary :all: ^ - --no-index ^ - --prefix %PREFIX% ^ - -vv - if %ERRORLEVEL% neq 0 exit 1 -) - -if NOT "%WHEELS_OUTPUT_FOLDER%"=="" ( - copy dist\mkl_umath*.whl %WHEELS_OUTPUT_FOLDER% - if %ERRORLEVEL% neq 0 exit 1 -) diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh deleted file mode 100644 index 4d25b512..00000000 --- a/conda-recipe/build.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -ex - -# This is necessary to help DPC++ find Intel libraries such as SVML, IRNG, etc in build prefix -export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${BUILD_PREFIX}/lib" - -# Intel LLVM must cooperate with compiler and sysroot from conda -echo "--gcc-toolchain=${BUILD_PREFIX} --sysroot=${BUILD_PREFIX}/${HOST}/sysroot -target ${HOST}" > icx_for_conda.cfg -ICXCFG="$(pwd)/icx_for_conda.cfg" -export ICXCFG - -read -r GLIBC_MAJOR GLIBC_MINOR <<< "$(conda list '^sysroot_linux-64$' \ - | tail -n 1 | awk '{print $2}' | grep -oP '\d+' | head -n 2 | tr '\n' ' ')" - -if [ -d "build" ]; then - rm -rf build -fi - -export CC=icx -export CXX=icpx - -# -wnx flags mean: --wheel --no-isolation --skip-dependency-check -# -Ccompile-args=-v makes ninja print full compiler commands (verbose build) -${PYTHON} -m build -w -n -x -Ccompile-args=-v - -${PYTHON} -m wheel tags --remove \ - --platform-tag "manylinux_${GLIBC_MAJOR}_${GLIBC_MINOR}_x86_64" \ - dist/mkl_umath*.whl - -${PYTHON} -m pip install dist/mkl_umath*.whl \ - --no-build-isolation \ - --no-deps \ - --only-binary :all: \ - --no-index \ - --prefix "${PREFIX}" \ - -vv - -# Copy wheel package -if [ -n "${WHEELS_OUTPUT_FOLDER}" ]; then - mkdir -p "${WHEELS_OUTPUT_FOLDER}" - cp dist/mkl_umath*.whl "${WHEELS_OUTPUT_FOLDER}" -fi diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml deleted file mode 100644 index 0f279420..00000000 --- a/conda-recipe/conda_build_config.yaml +++ /dev/null @@ -1,20 +0,0 @@ -numpy: - - '2' -c_compiler: # [linux] - - gcc # [linux] -cxx_compiler: # [linux] - - gxx # [linux] -cxx_compiler_version: # [linux] - - '14' # [linux] -c_stdlib: # [linux] - - sysroot # [linux] -c_stdlib_version: # [linux] - - '2.28' # [linux] -c_stdlib: # [win] - - vs # [win] -cxx_compiler: # [win] - - vs2022 # [win] -c_compiler: # [win] - - vs2022 # [win] -CL: - - /fp:precise # [win] diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml deleted file mode 100644 index 10679606..00000000 --- a/conda-recipe/meta.yaml +++ /dev/null @@ -1,67 +0,0 @@ -package: - name: mkl_umath - version: {{ GIT_DESCRIBE_TAG }} - -source: - path: ../ - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - script_env: - - WHEELS_OUTPUT_FOLDER - ignore_run_exports: - - blas - - numpy - - intel-openmp # [win] - -requirements: - build: - - {{ compiler('c') }} - - {{ compiler('cxx') }} - - {{ stdlib('c') }} - - {{ compiler('dpcpp') }} >=2024.2 # [not osx] - host: - - meson-python >=0.16.0 - - cmake - - ninja - - git - - cython >=3.1.0 - - python - - python-build >=1.2.2 - - mkl-devel - - intel-openmp - - intel-cmplr-lib-rt - - numpy - - wheel >=0.41.3 - - pip - run: - - python - - {{ pin_compatible('mkl') }} - - {{ pin_compatible('intel-openmp') }} - - {{ pin_compatible('intel-cmplr-lib-rt') }} - - numpy >=2 - -test: - requires: - - pytest - commands: - - pytest --pyargs mkl_umath - imports: - - mkl_umath - - mkl_umath._ufuncs - - mkl_umath._patch_numpy - -about: - home: http://github.com/IntelPython/mkl_umath - license: BSD-3-Clause - license_file: LICENSE.txt - summary: Universal functions for real and complex floating point arrays powered by Intel(R) Math Kernel Library Vector (Intel(R) MKL) and Intel(R) Short Vector Math Library (Intel(R) SVML) - description: | - LEGAL NOTICE: Use of this software package is subject to the - software license agreement (as set forth above, in the license section of - the installed Conda package and/or the README file) and all notices, - disclaimers or license terms for third party or open source software - included in or with the software. -

- EULA: BSD-3-Clause -

diff --git a/meson.build b/meson.build index c9312856..e9cbc8dc 100644 --- a/meson.build +++ b/meson.build @@ -112,6 +112,12 @@ else error('Unsupported system.') endif +# mkl-devel's CMake config is under Library on Windows, which CMake doesn't search +mkl_cmake_prefix = [] +if host_machine.system() == 'windows' + mkl_cmake_prefix += '-DCMAKE_PREFIX_PATH=' + (py.get_path('data') / 'Library') +endif + mkl_dep = dependency('MKL', method: 'cmake', modules: ['MKL::MKL'], cmake_args: [ @@ -119,7 +125,7 @@ mkl_dep = dependency('MKL', method: 'cmake', '-DMKL_LINK=dynamic', '-DMKL_THREADING=' + get_option('mkl_threading'), '-DMKL_INTERFACE=lp64' - ], + ] + mkl_cmake_prefix, required: true ) diff --git a/mkl_umath/tests/AGENTS.md b/mkl_umath/tests/AGENTS.md index a9f4d5c9..f78272b8 100644 --- a/mkl_umath/tests/AGENTS.md +++ b/mkl_umath/tests/AGENTS.md @@ -19,7 +19,7 @@ pytest mkl_umath/tests/ ``` ## CI integration -- Tests run in conda-package.yml workflow +- Tests run in the conda-package-cf.yml and intel-ci.yml workflows - Separate test jobs per Python version (3.10-3.15) - Linux + Windows platforms diff --git a/pyproject.toml b/pyproject.toml index ec1197ae..b6b347fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,9 +80,31 @@ extend-exclude = "(^|/)_vendored/" force-exclude = "(^|/)_vendored/" line-length = 80 -[tool.cython-lint] -ignore = ['E722'] # do not use bare 'except' -max-line-length = 80 +[tool.cibuildwheel] +build-verbosity = 1 + +[tool.cibuildwheel.linux] +# Intel's icx, as the conda recipe builds with, from Intel's repository +before-all = [ + "rpm --import https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB", + "printf '[oneAPI]\\nname=Intel oneAPI\\nbaseurl=https://yum.repos.intel.com/oneapi\\nenabled=1\\ngpgcheck=1\\nrepo_gpgcheck=1\\ngpgkey=https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB\\n' > /etc/yum.repos.d/oneAPI.repo", + "dnf install -y intel-oneapi-compiler-dpcpp-cpp" +] +environment = { CC = "/opt/intel/oneapi/compiler/latest/bin/icx" } +# MKL, OpenMP and icx's runtime come from wheels at run time, see [tool.gh-ci] +repair-wheel-command = "auditwheel repair --exclude libmkl_intel_lp64.so.3 --exclude libmkl_intel_thread.so.3 --exclude libmkl_core.so.3 --exclude libiomp5.so --exclude libimf.so --exclude libsvml.so --exclude libirng.so --exclude libintlc.so.5 -w {dest_dir} {wheel}" + +[tool.cibuildwheel.windows] +# MSVC's environment, as python.org builds CPython with, and icx from conda-forge +before-all = "python {project}/ci/install_icx.py C:/icx" +config-settings = { setup-args = "--vsenv" } +environment = { CC = "C:/icx/Library/bin/icx.exe", LIB = "C:/icx/Library/lib;$LIB", LINK = "/LIBPATH:C:/icx/Library/lib" } +# MKL, OpenMP and icx's runtime come from wheels at run time, see [tool.gh-ci] +repair-wheel-command = "delvewheel repair --no-dll mkl_umath_loops.dll;mkl_intel_thread.3.dll;mkl_core.3.dll;libiomp5md.dll;libmmd.dll;svml_dispmd.dll;libircmd.dll;libirngmd.dll -w {dest_dir} {wheel}" + +[tool.gh-ci] +# What the wheels need at run time, which conda packages get from the recipe +wheel-requires = ["mkl >=2026.2,<2027", "intel-openmp >=2026.2,<2027", "intel-cmplr-lib-rt >=2026.2,<2027"] [tool.isort] ensure_newline_before_comments = true