From c80c3e9979d175a22da152e86c9f63fe94770d3c Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Mon, 21 Sep 2026 14:32:45 -0500 Subject: [PATCH 1/4] Build the benchmarking environment from ci/environment.yml #1548 moved cartopy, matplotlib, spatialpandas and the rest of the geospatial/plotting stack out of the core dependencies and into the "geo" and "viz" extras. It did not touch benchmarks/asv.conf.json, and asv does not use the ci/environment.yml environment that runs it -- it built its own from "matrix", which listed xarray and netcdf4 and nothing else, and then pip-installed the wheel there, which since #1548 resolves core dependencies only. So two benchmarks have been reporting "failed" rather than a time, on every commit including main: mpas_ocean.GeoDataFrame.time_to_geodataframe OptionalDependencyNotFoundError: Failed to import: spatialpandas. mpas_ocean.MatplotlibConversion.time_dataarray_to_polycollection OptionalDependencyNotFoundError: Failed to import: cartopy, matplotlib. Point asv at ci/environment.yml instead. Benchmarks then run against the package set the project is actually tested against, and a dependency has to be added in one place rather than two -- which fixes the class of drift rather than the two instances of it that happen to be visible today. Conda now supplies the dependencies, so the project is installed with --no-deps, as the test job in ci.yml already does. Without that, pip would resolve the wheel's requirements from PyPI and, together with the --force-reinstall asv continuous needs (asv gh-1421), lay PyPI wheels over conda's builds on every commit. The matrix keeps only what the environment file does not carry. xarray and netcdf4 come from the file now. python-build, wheel and setuptools_scm are named explicitly because asv seeds an environment with wheel and pip only when it builds that environment without a file. Note that ci/environment.yml pins no Python version on purpose, and asv only injects "pythons" when there is no environment file, so conda now resolves the interpreter. Benchmarks move from 3.11 to whatever that resolves to, matching the environment asv itself runs in. A given comparison stays internally valid -- asv continuous installs both commits into the same environment -- but absolute timings are not comparable with results recorded before this. Co-Authored-By: Claude Opus 5 --- benchmarks/asv.conf.json | 56 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 31a43921d..4549e0489 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -34,7 +34,22 @@ // Customizable commands for installing and uninstalling the project. // See asv.conf.json documentation. - // "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}"], + // + // ``--no-deps``, because "conda_environment_file" below builds the + // environment from ci/environment.yml and conda has already put every + // dependency in place. Without it pip resolves the wheel's requirements from + // PyPI and, together with ``--force-reinstall``, lays PyPI wheels over + // conda's builds on every commit. This is the same reasoning as the + // ``python -m pip install . --no-deps`` the test job uses (ci.yml). + // + // ``--force-reinstall`` is kept from asv's own default command + // (``_install_project`` in asv/environment.py): ``asv continuous`` needs it + // because the project version does not necessarily change between the two + // commits it installs (asv gh-1421). It stays cheap here precisely because + // ``--no-deps`` confines it to uxarray itself. + "install_command": [ + "in-dir={env_dir} python -m pip install {wheel_file} --no-deps --force-reinstall" + ], // "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"], // List of branches to benchmark. If not provided, defaults to "master" @@ -57,6 +72,30 @@ "environment_type": "conda", "conda_channels": ["conda-forge"], + // Build the benchmarking environment from the same file as the rest of CI, + // so benchmarks run against the package set the project is actually tested + // against and a dependency only has to be added in one place. + // + // Previously asv built this environment from "matrix" alone, which listed + // xarray and netcdf4 and nothing else. #1548 then moved cartopy, matplotlib + // and spatialpandas out of the core dependencies into the "geo" and "viz" + // extras; the wheel install stopped pulling them in, and + // ``to_geodataframe()`` and ``to_polycollection()`` went from being measured + // to reporting "failed" with an OptionalDependencyNotFoundError. Tracking + // ci/environment.yml removes that whole class of drift rather than the two + // instances of it. + // + // The path is relative to the directory asv runs in, which is this one -- + // see ``working-directory: ${{ env.ASV_DIR }}`` in + // .github/workflows/asv-benchmarking-pr.yml. + // + // Note that ci/environment.yml pins no Python version on purpose (ci.yml + // supplies one per matrix entry through micromamba's create-args), and asv + // only injects "pythons" into an environment it builds without a file. So + // conda resolves the interpreter here, and "pythons" below just names the + // environment; it does not select one. + "conda_environment_file": "../ci/environment.yml", + // timeout in seconds for installing any dependencies in environment // defaults to 10 min "install_timeout": 600, @@ -90,10 +129,21 @@ // new environments. A value of ``null`` means that the variable // will not be set for the current combination. // + // Only what ci/environment.yml does not already provide; asv applies these + // as a ``conda env update`` over the environment built from that file. + // xarray and netcdf4 used to be listed here and now come from the file. + // + // python-build and wheel are what "build_command" below runs on, and + // setuptools_scm is what it builds with, since its second command passes + // --no-build-isolation. They are named here rather than left implicit + // because asv seeds an environment with wheel and pip only when it is + // building that environment without a file (``_setup`` in + // asv/plugins/conda.py), which is no longer the case. pip itself is in + // ci/environment.yml. pyfma comes from PyPI. "matrix": { + "python-build": [""], + "wheel": [""], "setuptools_scm": [""], - "xarray": [""], - "netcdf4": [""], "pip+pyfma": [""] }, From d7c774a461675b3d28d7556464443e5b4cdb68bd Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Mon, 21 Sep 2026 14:40:16 -0500 Subject: [PATCH 2/4] Cache the asv environment between benchmark runs asv does not benchmark in the uxarray_build environment set up by this workflow; it builds its own under benchmarks/env/, and every run gets a fresh runner, so that environment is solved and linked from scratch each time. Pointing asv at ci/environment.yml did not change that -- it changed which packages go in, not how often they go in. Restore the directory instead. Once it is present, Environment.create() returns without calling _setup() as soon as asv-env-info.json matches the configured tool, python and requirements (asv/environment.py), so the whole solve is skipped. That check does not inspect the installed packages, so the key carries everything that determines them: ci/environment.yml for the package set and asv.conf.json for the matrix and for which environment file is in use. There are deliberately no restore-keys -- a prefix match would restore an environment built from an older ci/environment.yml and asv would reuse it without noticing, which is the failure this is meant to avoid. Co-Authored-By: Claude Opus 5 --- .github/workflows/asv-benchmarking-pr.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/asv-benchmarking-pr.yml b/.github/workflows/asv-benchmarking-pr.yml index cae4aa6f0..462d6e55f 100644 --- a/.github/workflows/asv-benchmarking-pr.yml +++ b/.github/workflows/asv-benchmarking-pr.yml @@ -43,6 +43,26 @@ jobs: python-build mamba + # asv does not benchmark in the environment above; it builds its own under + # benchmarks/env/ (from ci/environment.yml, see benchmarks/asv.conf.json) + # and that one is rebuilt from scratch on every run, because each run gets + # a fresh runner. Restoring the directory skips the whole solve-and-link: + # once it is there, Environment.create() returns without calling _setup() + # as soon as asv-env-info.json matches the configured tool, python and + # requirements (asv/environment.py). + # + # That check does not look at the installed packages, so the key has to + # carry everything that determines them: ci/environment.yml for the package + # set, asv.conf.json for the matrix and for which environment file is used. + # Deliberately no restore-keys -- a prefix match would restore an + # environment built from an older ci/environment.yml, and asv would reuse + # it without noticing, which is the failure this is meant to avoid. + - name: Cache asv environment + uses: actions/cache@v4 + with: + path: ${{ env.ASV_DIR }}/env + key: "asv-env-${{runner.os}}-${{runner.arch}}-${{hashFiles(env.CONDA_ENV_FILE, 'benchmarks/asv.conf.json')}}" + - name: Run Benchmarks shell: bash -l {0} id: benchmark From 31256623c7e4b956d0c3259c56b7ed71157050aa Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Mon, 21 Sep 2026 17:24:34 -0500 Subject: [PATCH 3/4] Deslop ASV optional deps support --- .github/workflows/asv-benchmarking-pr.yml | 16 +------ benchmarks/asv.conf.json | 51 +++-------------------- 2 files changed, 8 insertions(+), 59 deletions(-) diff --git a/.github/workflows/asv-benchmarking-pr.yml b/.github/workflows/asv-benchmarking-pr.yml index 462d6e55f..7b11775fb 100644 --- a/.github/workflows/asv-benchmarking-pr.yml +++ b/.github/workflows/asv-benchmarking-pr.yml @@ -43,20 +43,8 @@ jobs: python-build mamba - # asv does not benchmark in the environment above; it builds its own under - # benchmarks/env/ (from ci/environment.yml, see benchmarks/asv.conf.json) - # and that one is rebuilt from scratch on every run, because each run gets - # a fresh runner. Restoring the directory skips the whole solve-and-link: - # once it is there, Environment.create() returns without calling _setup() - # as soon as asv-env-info.json matches the configured tool, python and - # requirements (asv/environment.py). - # - # That check does not look at the installed packages, so the key has to - # carry everything that determines them: ci/environment.yml for the package - # set, asv.conf.json for the matrix and for which environment file is used. - # Deliberately no restore-keys -- a prefix match would restore an - # environment built from an older ci/environment.yml, and asv would reuse - # it without noticing, which is the failure this is meant to avoid. + # asv rebuilds its own env under benchmarks/env/ each run; caching skips the + # solve. No restore-keys: asv reuses a restored env without checking it. - name: Cache asv environment uses: actions/cache@v4 with: diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 4549e0489..ffb71769c 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -35,18 +35,8 @@ // Customizable commands for installing and uninstalling the project. // See asv.conf.json documentation. // - // ``--no-deps``, because "conda_environment_file" below builds the - // environment from ci/environment.yml and conda has already put every - // dependency in place. Without it pip resolves the wheel's requirements from - // PyPI and, together with ``--force-reinstall``, lays PyPI wheels over - // conda's builds on every commit. This is the same reasoning as the - // ``python -m pip install . --no-deps`` the test job uses (ci.yml). - // - // ``--force-reinstall`` is kept from asv's own default command - // (``_install_project`` in asv/environment.py): ``asv continuous`` needs it - // because the project version does not necessarily change between the two - // commits it installs (asv gh-1421). It stays cheap here precisely because - // ``--no-deps`` confines it to uxarray itself. + // --no-deps: conda already installed every dependency from ci/environment.yml. + // --force-reinstall is asv's default; the version may not change (asv gh-1421). "install_command": [ "in-dir={env_dir} python -m pip install {wheel_file} --no-deps --force-reinstall" ], @@ -72,28 +62,8 @@ "environment_type": "conda", "conda_channels": ["conda-forge"], - // Build the benchmarking environment from the same file as the rest of CI, - // so benchmarks run against the package set the project is actually tested - // against and a dependency only has to be added in one place. - // - // Previously asv built this environment from "matrix" alone, which listed - // xarray and netcdf4 and nothing else. #1548 then moved cartopy, matplotlib - // and spatialpandas out of the core dependencies into the "geo" and "viz" - // extras; the wheel install stopped pulling them in, and - // ``to_geodataframe()`` and ``to_polycollection()`` went from being measured - // to reporting "failed" with an OptionalDependencyNotFoundError. Tracking - // ci/environment.yml removes that whole class of drift rather than the two - // instances of it. - // - // The path is relative to the directory asv runs in, which is this one -- - // see ``working-directory: ${{ env.ASV_DIR }}`` in - // .github/workflows/asv-benchmarking-pr.yml. - // - // Note that ci/environment.yml pins no Python version on purpose (ci.yml - // supplies one per matrix entry through micromamba's create-args), and asv - // only injects "pythons" into an environment it builds without a file. So - // conda resolves the interpreter here, and "pythons" below just names the - // environment; it does not select one. + // Same package set as the rest of CI, so benchmarks cannot drift from it: + // #1548 made cartopy, matplotlib and spatialpandas optional and broke two. "conda_environment_file": "../ci/environment.yml", // timeout in seconds for installing any dependencies in environment @@ -129,17 +99,8 @@ // new environments. A value of ``null`` means that the variable // will not be set for the current combination. // - // Only what ci/environment.yml does not already provide; asv applies these - // as a ``conda env update`` over the environment built from that file. - // xarray and netcdf4 used to be listed here and now come from the file. - // - // python-build and wheel are what "build_command" below runs on, and - // setuptools_scm is what it builds with, since its second command passes - // --no-build-isolation. They are named here rather than left implicit - // because asv seeds an environment with wheel and pip only when it is - // building that environment without a file (``_setup`` in - // asv/plugins/conda.py), which is no longer the case. pip itself is in - // ci/environment.yml. pyfma comes from PyPI. + // Only what ci/environment.yml lacks: the build tools, which asv seeds itself + // only when building an environment without a file, plus pyfma from PyPI. "matrix": { "python-build": [""], "wheel": [""], From 30153e781369188c8b7b8d72cb42660e42f68a05 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Tue, 22 Sep 2026 16:07:03 -0500 Subject: [PATCH 4/4] actions cache v6 --- .github/workflows/asv-benchmarking-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/asv-benchmarking-pr.yml b/.github/workflows/asv-benchmarking-pr.yml index 7b11775fb..62dcd8961 100644 --- a/.github/workflows/asv-benchmarking-pr.yml +++ b/.github/workflows/asv-benchmarking-pr.yml @@ -46,7 +46,7 @@ jobs: # asv rebuilds its own env under benchmarks/env/ each run; caching skips the # solve. No restore-keys: asv reuses a restored env without checking it. - name: Cache asv environment - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: ${{ env.ASV_DIR }}/env key: "asv-env-${{runner.os}}-${{runner.arch}}-${{hashFiles(env.CONDA_ENV_FILE, 'benchmarks/asv.conf.json')}}"