From 71ef47bcafe49c4e3859f4359259f8241c63e2a5 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Mon, 5 Oct 2026 13:01:15 -0500 Subject: [PATCH 01/11] CI clears numba cache --- .pre-commit-config.yaml | 9 ++++++ ci/clean_numba_cache.py | 56 ++++++++++++++++++++++++++++++++++++++ docs/contributing.rst | 10 +++++++ uxarray/grid/utils.py | 2 +- uxarray/utils/computing.py | 6 ++++ 5 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 ci/clean_numba_cache.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c11e500cc..5cd15ce3a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,3 +25,12 @@ repos: - id: ruff-format name: format with ruff types_or: [ python, pyi, jupyter ] + +- repo: local + hooks: + - id: clean-numba-cache + name: clear stale numba caches + entry: python ci/clean_numba_cache.py + language: python + files: ^uxarray/(utils/(computing|numba_math)|grid/(angles|arcs|area|coordinates|geometry|intersections|point_in_face|utils))\.py$ + pass_filenames: false diff --git a/ci/clean_numba_cache.py b/ci/clean_numba_cache.py new file mode 100644 index 000000000..c245f78b8 --- /dev/null +++ b/ci/clean_numba_cache.py @@ -0,0 +1,56 @@ +"""Delete numba's on-disk cache for ``uxarray``. + +Numba stamps each ``cache=True`` index (``*.nbi``) with the ``(mtime, size)`` of the +source file that *defines* the function, and nothing else. A kernel that calls a jitted +function from another module has that callee's code linked into its own cached object, +so editing the callee leaves the caller's cache entry "valid" with the old body baked in. + +Run this after editing any shared jitted module, or let the pre-commit hook do it:: + + python ci/clean_numba_cache.py [--dry-run] +""" + +import argparse +import os +from pathlib import Path + +PACKAGE_DIR = Path(__file__).resolve().parent.parent / "uxarray" +CACHE_SUFFIXES = {".nbi", ".nbc"} + + +def _cache_files(): + """Yield numba cache files stored next to the package's sources.""" + for path in PACKAGE_DIR.rglob("__pycache__/*"): + if path.suffix in CACHE_SUFFIXES: + yield path + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument( + "-n", + "--dry-run", + action="store_true", + help="list what would be removed without deleting anything", + ) + args = parser.parse_args(argv) + + files = sorted(set(_cache_files())) + for path in files: + if args.dry_run: + print(path) + else: + path.unlink(missing_ok=True) + + if os.environ.get("NUMBA_CACHE_DIR"): + # Numba files caches there under a hash of each source directory, so they cannot + # be told apart from other projects' entries; leave them alone rather than guess. + print("NUMBA_CACHE_DIR is set: clear it by hand, it is not touched here") + + verb = "would remove" if args.dry_run else "removed" + print(f"{verb} {len(files)} numba cache file(s)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/contributing.rst b/docs/contributing.rst index cffde8e8f..a07d30b70 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -360,6 +360,16 @@ following command:: $ pre-commit run --all-files +One of the hooks, ``clean-numba-cache``, deletes numba's on-disk kernel cache when you stage a +change to one of the shared jitted modules (``uxarray/utils/computing.py``, ``numba_math.py`` +and the ``uxarray/grid`` modules that call them). Numba only checks a cached kernel against +the file that defines it, so a kernel that calls a jitted function from another file would +otherwise keep running the old code after that function is edited. The hook only fires at +commit time, so after editing one of these files, run the same cleanup by hand before you +test the change:: + + $ python ci/clean_numba_cache.py + 3.5. Use Feature Branches ------------------------- diff --git a/uxarray/grid/utils.py b/uxarray/grid/utils.py index 4d01e5049..472f47e68 100644 --- a/uxarray/grid/utils.py +++ b/uxarray/grid/utils.py @@ -526,7 +526,7 @@ def setter(self, value): # # NOTE: these are inlined into ``cache=True`` kernels in another module, and numba stamps its # cache against the defining file alone, so editing them does not invalidate a caller's cached -# object. Clear ``uxarray/grid/__pycache__/*.nbi *.nbc`` after changing anything here. +# object. Run ``python ci/clean_numba_cache.py`` after changing anything here. @njit(cache=True) diff --git a/uxarray/utils/computing.py b/uxarray/utils/computing.py index aff800372..098c39ee5 100644 --- a/uxarray/utils/computing.py +++ b/uxarray/utils/computing.py @@ -8,6 +8,12 @@ hardware FMA when one is available (validated bit-exact at import time) and falls back to the Veltkamp split otherwise, so there is no hard FMA dependency. +.. warning:: + Every primitive here is inlined into ``cache=True`` kernels in other modules. Numba + stamps a cache entry against the file that *defines* the kernel only, + so after editing this file those callers keep running the old body from their cache, + with no warning. Run ``python ci/clean_numba_cache.py`` before testing changes here. + Python/Numba port of the AccuSphGeom C++ library (EFT tier only; the adaptive Shewchuk predicate and exact-arithmetic fallback tiers are not ported): From 92cc4fc6d107923995fd4a659bc302cbb8b8951e Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Mon, 5 Oct 2026 14:23:15 -0500 Subject: [PATCH 02/11] Guard numba flags the compensated EFT kernels depend on Add a test that forbids fastmath anywhere code from uxarray.utils.computing is compiled, and requires error_model="numpy" on acc_sqrt_re, _accux_gca, _accux_constlat and every kernel they are inlined into. An inline="always" callee is compiled with its caller's flags, so the test follows numba's inline call chains rather than only checking the definitions. Co-Authored-By: Claude Opus 5.5 --- test/grid/geometry/test_numba_flags.py | 112 +++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 test/grid/geometry/test_numba_flags.py diff --git a/test/grid/geometry/test_numba_flags.py b/test/grid/geometry/test_numba_flags.py new file mode 100644 index 000000000..d283e80f2 --- /dev/null +++ b/test/grid/geometry/test_numba_flags.py @@ -0,0 +1,112 @@ +"""Guard the numba flags that the compensated (EFT) kernels depend on. + +``fastmath`` lets LLVM reassociate ``e = (a - (s - bp)) + (b - bp)`` to 0, silently undoing +every compensated result from ``uxarray.utils.computing``. ``acc_sqrt_re``, ``_accux_gca`` +and ``_accux_constlat`` need ``error_model="numpy"`` so that 1/0 and sqrt(<0) give inf/nan +for the status layer to mask, instead of raising. + +An ``inline="always"`` callee is compiled with its caller's flags, so both checks apply to +every kernel this code is inlined into, not just where it is defined. +""" + +import ast +import importlib +import inspect +import pkgutil +import sys +import textwrap +from pathlib import Path + +import pytest +from numba.core.registry import CPUDispatcher + +import uxarray + +EFT_MODULE = "uxarray.utils.computing" +NUMPY_ERROR_MODEL_KERNELS = { + "uxarray.utils.computing.acc_sqrt_re", + "uxarray.grid.intersections._accux_gca", + "uxarray.grid.intersections._accux_constlat", +} + + +def _qualname(kernel): + return f"{kernel.py_func.__module__}.{kernel.py_func.__name__}" + + +def _callees(kernel): + """Yield the jitted functions called by name in ``kernel``'s body.""" + namespace = kernel.py_func.__globals__ + for node in ast.walk(ast.parse(textwrap.dedent(inspect.getsource(kernel.py_func)))): + if not isinstance(node, ast.Call): + continue + func = node.func + if isinstance(func, ast.Name): + target = namespace.get(func.id) + elif isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name): + target = getattr(namespace.get(func.value.id), func.attr, None) + else: + continue + if isinstance(target, CPUDispatcher): + yield target + + +def _compiled_into(kernel): + """Names of ``kernel`` and every kernel inlined into it, all built with its flags.""" + names, stack = {_qualname(kernel)}, [kernel] + while stack: + for callee in _callees(stack.pop()): + name = _qualname(callee) + if callee.targetoptions.get("inline") == "always" and name not in names: + names.add(name) + stack.append(callee) + return names + + +@pytest.fixture(scope="module") +def kernels(): + """Map each module-level kernel in uxarray to ``(kernel, names compiled into it)``.""" + found = {} + for info in pkgutil.walk_packages(uxarray.__path__, "uxarray."): + try: + module = importlib.import_module(info.name) + except ImportError: # missing optional dependency: its kernels can't run either + continue + for obj in vars(module).values(): + if isinstance(obj, CPUDispatcher) and obj.py_func.__module__ == info.name: + found[_qualname(obj)] = (obj, _compiled_into(obj)) + assert NUMPY_ERROR_MODEL_KERNELS <= found.keys() # else the checks pass vacuously + return found + + +def test_numpy_error_model(kernels): + offenders = sorted( + name + for name, (kernel, compiled) in kernels.items() + if compiled & NUMPY_ERROR_MODEL_KERNELS + and kernel.targetoptions.get("error_model") != "numpy" + ) + assert not offenders, f"need error_model='numpy' to mask inf/nan: {offenders}" + + +def test_no_fastmath(kernels): + eft = { + name: kernel + for name, (kernel, compiled) in kernels.items() + if any(n.startswith(EFT_MODULE + ".") for n in compiled) + } + offenders = [ + name for name, kernel in eft.items() if kernel.targetoptions.get("fastmath") + ] + # Also scan the source, for kernels the import walk can't see: nested ones and the + # ``two_prod`` variant not selected on this machine. + for module in {kernel.py_func.__module__ for kernel in eft.values()}: + tree = ast.parse(Path(sys.modules[module].__file__).read_text()) + offenders += [ + f"{module}:{node.lineno}" + for node in ast.walk(tree) + if isinstance(node, ast.keyword) and node.arg == "fastmath" + ] + assert not offenders, ( + f"fastmath would cancel the EFT error terms: {sorted(offenders)}" + ) From 355a8cb1eb56985395daa7d9255c1ec857edda1e Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 23 Oct 2026 13:58:52 -0500 Subject: [PATCH 03/11] h1h2: some comments --- ci/clean_numba_cache.py | 3 ++- test/grid/geometry/test_numba_flags.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ci/clean_numba_cache.py b/ci/clean_numba_cache.py index c245f78b8..c2cc42504 100644 --- a/ci/clean_numba_cache.py +++ b/ci/clean_numba_cache.py @@ -45,7 +45,8 @@ def main(argv=None): if os.environ.get("NUMBA_CACHE_DIR"): # Numba files caches there under a hash of each source directory, so they cannot # be told apart from other projects' entries; leave them alone rather than guess. - print("NUMBA_CACHE_DIR is set: clear it by hand, it is not touched here") + print("""WARNING: NUMBA_CACHE_DIR is set. To avoid corrupting caches of external projects, + no caches in this directory (NUMBA_CACHE_DIR) were deleted. Please clear it by hand""") verb = "would remove" if args.dry_run else "removed" print(f"{verb} {len(files)} numba cache file(s)") diff --git a/test/grid/geometry/test_numba_flags.py b/test/grid/geometry/test_numba_flags.py index d283e80f2..43edf7abe 100644 --- a/test/grid/geometry/test_numba_flags.py +++ b/test/grid/geometry/test_numba_flags.py @@ -40,13 +40,18 @@ def _callees(kernel): for node in ast.walk(ast.parse(textwrap.dedent(inspect.getsource(kernel.py_func)))): if not isinstance(node, ast.Call): continue + func = node.func + # A direct function call (e.g., print()) if isinstance(func, ast.Name): target = namespace.get(func.id) + # An attribute method call (e.g. val.print()) elif isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name): target = getattr(namespace.get(func.value.id), func.attr, None) else: continue + + # Only consider Numba JIT func calls if isinstance(target, CPUDispatcher): yield target From 4a8aa285c584c0d08958b3c2a8fe140cc87452b1 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 23 Oct 2026 14:06:32 -0500 Subject: [PATCH 04/11] h1h2: fix false positive on --- test/grid/geometry/test_numba_flags.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/grid/geometry/test_numba_flags.py b/test/grid/geometry/test_numba_flags.py index 43edf7abe..71be250b3 100644 --- a/test/grid/geometry/test_numba_flags.py +++ b/test/grid/geometry/test_numba_flags.py @@ -104,13 +104,16 @@ def test_no_fastmath(kernels): name for name, kernel in eft.items() if kernel.targetoptions.get("fastmath") ] # Also scan the source, for kernels the import walk can't see: nested ones and the - # ``two_prod`` variant not selected on this machine. + # ``two_prod`` variant not selected on this machine. Only a literal ``False`` is known + # to be safe; anything else (``True``, a flag set, a variable) is flagged. for module in {kernel.py_func.__module__ for kernel in eft.values()}: tree = ast.parse(Path(sys.modules[module].__file__).read_text()) offenders += [ f"{module}:{node.lineno}" for node in ast.walk(tree) - if isinstance(node, ast.keyword) and node.arg == "fastmath" + if isinstance(node, ast.keyword) + and node.arg == "fastmath" + and not (isinstance(node.value, ast.Constant) and node.value.value is False) ] assert not offenders, ( f"fastmath would cancel the EFT error terms: {sorted(offenders)}" From d234d35ac05611e138c48c584829c0448e384b1c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 19:55:07 +0000 Subject: [PATCH 05/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- ci/clean_numba_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/clean_numba_cache.py b/ci/clean_numba_cache.py index c2cc42504..aa5daa647 100644 --- a/ci/clean_numba_cache.py +++ b/ci/clean_numba_cache.py @@ -45,7 +45,7 @@ def main(argv=None): if os.environ.get("NUMBA_CACHE_DIR"): # Numba files caches there under a hash of each source directory, so they cannot # be told apart from other projects' entries; leave them alone rather than guess. - print("""WARNING: NUMBA_CACHE_DIR is set. To avoid corrupting caches of external projects, + print("""WARNING: NUMBA_CACHE_DIR is set. To avoid corrupting caches of external projects, no caches in this directory (NUMBA_CACHE_DIR) were deleted. Please clear it by hand""") verb = "would remove" if args.dry_run else "removed" From 5e7c6641ed52ac1d981474c62e0f1488e6538930 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Tue, 27 Oct 2026 18:14:01 -0500 Subject: [PATCH 06/11] h1h2: more comments --- test/grid/geometry/test_numba_flags.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/test/grid/geometry/test_numba_flags.py b/test/grid/geometry/test_numba_flags.py index 71be250b3..8ddfee02f 100644 --- a/test/grid/geometry/test_numba_flags.py +++ b/test/grid/geometry/test_numba_flags.py @@ -83,18 +83,28 @@ def kernels(): assert NUMPY_ERROR_MODEL_KERNELS <= found.keys() # else the checks pass vacuously return found - +# For each inlined caller kernel, look at the included/invoked callee subkernels and +# filter out caller kernels that don't use numpy error model where at least one of +# the callee subkernels uses the numpy error model. i.e., instances of +# InlinedCallerKernelNotUsingNumpyErrorModel where CalleeSubKernelUsesNumpyErrorModel def test_numpy_error_model(kernels): + """Test that functions use error_model==numpy if it calls inline + functions with error_model == numpy. + """ + offenders = sorted( name - for name, (kernel, compiled) in kernels.items() - if compiled & NUMPY_ERROR_MODEL_KERNELS - and kernel.targetoptions.get("error_model") != "numpy" + for name, (kernel, is_compiled) in kernels.items() + if is_compiled & NUMPY_ERROR_MODEL_KERNELS # Compiled kernel is using numpy error model + and kernel.targetoptions.get("error_model") != "numpy" # ...but the current kernel is *not*! ) - assert not offenders, f"need error_model='numpy' to mask inf/nan: {offenders}" + assert not offenders, (f"Missing error_model='numpy' in function decorations of : {offenders}. " + f"Error model numpy is needed for some kernels used in these functions " + f"(to avoid exceptions on inf/nan)") def test_no_fastmath(kernels): + """Test that functions don't use fastmath if they call inline functions""" eft = { name: kernel for name, (kernel, compiled) in kernels.items() From c6cf41acde3f2e7d6d4594abec88127027b73f67 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Mon, 16 Nov 2026 11:07:42 -0600 Subject: [PATCH 07/11] Address review comments on numba flag tests - Rename _compiled_into to _get_inlined_funcs_in_call_tree. - Move the NUMPY_ERROR_MODEL_KERNELS presence check from the fixture into test_numpy_error_model, with a message, and add a matching guard to test_no_fastmath so neither test can pass vacuously. - Explain why NUMPY_ERROR_MODEL_KERNELS lists the source kernels by hand. Co-Authored-By: Claude Opus 5.5 --- test/grid/geometry/test_numba_flags.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/grid/geometry/test_numba_flags.py b/test/grid/geometry/test_numba_flags.py index 8ddfee02f..779d9d0a6 100644 --- a/test/grid/geometry/test_numba_flags.py +++ b/test/grid/geometry/test_numba_flags.py @@ -23,6 +23,11 @@ import uxarray EFT_MODULE = "uxarray.utils.computing" +# The kernels whose own arithmetic relies on 1/0 -> inf and sqrt(<0) -> nan. This is the +# list of *sources*: the tests scan every kernel in uxarray and flag any one these are +# inlined into that lacks error_model="numpy". It has to be maintained by hand, because +# a kernel's own flags can't tell us this: e.g. ``_try_gca_gca_intersection`` sets +# error_model="numpy" only because it inlines ``_accux_gca``, not because it needs it. NUMPY_ERROR_MODEL_KERNELS = { "uxarray.utils.computing.acc_sqrt_re", "uxarray.grid.intersections._accux_gca", @@ -56,7 +61,7 @@ def _callees(kernel): yield target -def _compiled_into(kernel): +def _get_inlined_funcs_in_call_tree(kernel): """Names of ``kernel`` and every kernel inlined into it, all built with its flags.""" names, stack = {_qualname(kernel)}, [kernel] while stack: @@ -79,8 +84,7 @@ def kernels(): continue for obj in vars(module).values(): if isinstance(obj, CPUDispatcher) and obj.py_func.__module__ == info.name: - found[_qualname(obj)] = (obj, _compiled_into(obj)) - assert NUMPY_ERROR_MODEL_KERNELS <= found.keys() # else the checks pass vacuously + found[_qualname(obj)] = (obj, _get_inlined_funcs_in_call_tree(obj)) return found # For each inlined caller kernel, look at the included/invoked callee subkernels and @@ -91,6 +95,10 @@ def test_numpy_error_model(kernels): """Test that functions use error_model==numpy if it calls inline functions with error_model == numpy. """ + # If a source kernel is renamed, moved or fails to import, nothing would match it + # below and this test would pass without checking anything. + missing = NUMPY_ERROR_MODEL_KERNELS - kernels.keys() + assert not missing, f"NUMPY_ERROR_MODEL_KERNELS names unknown kernels: {missing}" offenders = sorted( name @@ -110,6 +118,7 @@ def test_no_fastmath(kernels): for name, (kernel, compiled) in kernels.items() if any(n.startswith(EFT_MODULE + ".") for n in compiled) } + assert eft, f"No kernels found in {EFT_MODULE}; was it renamed?" # else vacuous offenders = [ name for name, kernel in eft.items() if kernel.targetoptions.get("fastmath") ] From 9020f89c289a4d4eb4cde4513165b0c1ef50e35d Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Mon, 16 Nov 2026 11:20:55 -0600 Subject: [PATCH 08/11] Expand documentation of numba flag tests Explain why inlined EFT kernels need error_model="numpy" and no fastmath, with verified acc_sqrt_re examples, and correct the claim that sqrt of a negative raises under the default error model (only division does). Co-Authored-By: Claude Opus 5.5 --- test/grid/geometry/test_numba_flags.py | 51 ++++++++++++++------------ 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/test/grid/geometry/test_numba_flags.py b/test/grid/geometry/test_numba_flags.py index 779d9d0a6..4fc8623a9 100644 --- a/test/grid/geometry/test_numba_flags.py +++ b/test/grid/geometry/test_numba_flags.py @@ -1,12 +1,21 @@ -"""Guard the numba flags that the compensated (EFT) kernels depend on. - -``fastmath`` lets LLVM reassociate ``e = (a - (s - bp)) + (b - bp)`` to 0, silently undoing -every compensated result from ``uxarray.utils.computing``. ``acc_sqrt_re``, ``_accux_gca`` -and ``_accux_constlat`` need ``error_model="numpy"`` so that 1/0 and sqrt(<0) give inf/nan -for the status layer to mask, instead of raising. - -An ``inline="always"`` callee is compiled with its caller's flags, so both checks apply to -every kernel this code is inlined into, not just where it is defined. +"""Guard the numba flags that uxarray's compensated (EFT) kernels depend on. + +A kernel declared ``inline="always"`` is compiled with its *caller's* flags, not its own, +so every kernel it is inlined into, directly or transitively, needs the same flags. +Taking ``acc_sqrt_re`` (``uxarray.utils.computing``) as the example: + +- ``error_model="numpy"``: division follows IEEE 754, so ``acc_sqrt_re(0.0)`` returns + ``(0.0, nan)`` (the correction is 0/0) and the status layer's isfinite mask rejects it. + Under numba's default ``error_model="python"`` the same call raises ZeroDivisionError. + (A negative input gives ``(nan, nan)`` under either model; only division raises.) +- no ``fastmath``: compensated arithmetic relies on IEEE 754 forbidding reassociation. + ``fastmath`` lets LLVM simplify two_sum's ``e = (a - (s - bp)) + (b - bp)``, with + ``s = a + b``, to 0, discarding the rounding error the term exists to recover: + ``acc_sqrt_re(10.0)`` returns correction -1.9e-16 normally, 0.0 under fastmath. + +``test_numpy_error_model`` checks every uxarray kernel that inlines one of +``NUMPY_ERROR_MODEL_KERNELS``; ``test_no_fastmath`` checks every kernel that inlines +anything from ``uxarray.utils.computing``. """ import ast @@ -23,7 +32,7 @@ import uxarray EFT_MODULE = "uxarray.utils.computing" -# The kernels whose own arithmetic relies on 1/0 -> inf and sqrt(<0) -> nan. This is the +# The kernels whose own arithmetic relies on 1/0 -> inf and 0/0 -> nan. This is the # list of *sources*: the tests scan every kernel in uxarray and flag any one these are # inlined into that lacks error_model="numpy". It has to be maintained by hand, because # a kernel's own flags can't tell us this: e.g. ``_try_gca_gca_intersection`` sets @@ -87,14 +96,9 @@ def kernels(): found[_qualname(obj)] = (obj, _get_inlined_funcs_in_call_tree(obj)) return found -# For each inlined caller kernel, look at the included/invoked callee subkernels and -# filter out caller kernels that don't use numpy error model where at least one of -# the callee subkernels uses the numpy error model. i.e., instances of -# InlinedCallerKernelNotUsingNumpyErrorModel where CalleeSubKernelUsesNumpyErrorModel + def test_numpy_error_model(kernels): - """Test that functions use error_model==numpy if it calls inline - functions with error_model == numpy. - """ + """Every kernel that inlines a NUMPY_ERROR_MODEL_KERNELS entry sets error_model="numpy".""" # If a source kernel is renamed, moved or fails to import, nothing would match it # below and this test would pass without checking anything. missing = NUMPY_ERROR_MODEL_KERNELS - kernels.keys() @@ -103,16 +107,17 @@ def test_numpy_error_model(kernels): offenders = sorted( name for name, (kernel, is_compiled) in kernels.items() - if is_compiled & NUMPY_ERROR_MODEL_KERNELS # Compiled kernel is using numpy error model - and kernel.targetoptions.get("error_model") != "numpy" # ...but the current kernel is *not*! + if is_compiled & NUMPY_ERROR_MODEL_KERNELS # inlines a kernel that needs it... + and kernel.targetoptions.get("error_model") != "numpy" # ...but doesn't set it + ) + assert not offenders, ( + f"Missing error_model='numpy' on {offenders}: they inline kernels that rely on " + f"1/0 -> inf and 0/0 -> nan, which raise ZeroDivisionError under the default model" ) - assert not offenders, (f"Missing error_model='numpy' in function decorations of : {offenders}. " - f"Error model numpy is needed for some kernels used in these functions " - f"(to avoid exceptions on inf/nan)") def test_no_fastmath(kernels): - """Test that functions don't use fastmath if they call inline functions""" + """No kernel that inlines EFT code from uxarray.utils.computing enables fastmath.""" eft = { name: kernel for name, (kernel, compiled) in kernels.items() From 0e40fc201f654e3173ea9d61075f905404616598 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Tue, 17 Nov 2026 13:48:37 -0600 Subject: [PATCH 09/11] Expand the fastmath assertion message in numba flag tests Say why fastmath is rejected, what the offender entries are (kernels or module:line of a non-False fastmath keyword) and how to fix it. Co-Authored-By: Claude Opus 5.5 --- test/grid/geometry/test_numba_flags.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/grid/geometry/test_numba_flags.py b/test/grid/geometry/test_numba_flags.py index 4fc8623a9..49245dda1 100644 --- a/test/grid/geometry/test_numba_flags.py +++ b/test/grid/geometry/test_numba_flags.py @@ -140,5 +140,7 @@ def test_no_fastmath(kernels): and not (isinstance(node.value, ast.Constant) and node.value.value is False) ] assert not offenders, ( - f"fastmath would cancel the EFT error terms: {sorted(offenders)}" + f"fastmath enabled on {sorted(offenders)} (kernels, or module:line of a fastmath= " + f"that isn't a literal False): they inline EFT code from {EFT_MODULE}, whose " + f"error terms fastmath cancels to 0. Remove fastmath or set it to False." ) From 1607aec5c0f151d142d86e2d0e93b85098dd5b68 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Wed, 18 Nov 2026 17:24:30 -0600 Subject: [PATCH 10/11] Clear stale numba caches at pytest startup instead of in pre-commit The pre-commit hook cleared the cache on every `pre-commit run --all-files`, fired after testing rather than before, and never helped anyone pulling the branch or CI. pytest_sessionstart now hashes the shared jitted modules and clears the cache only when they differ from the last run, so a warm cache survives unrelated work and checkouts that change those modules are caught on any machine. ci/clean_numba_cache.py remains for use outside pytest. Co-Authored-By: Claude Opus 5.5 --- .pre-commit-config.yaml | 9 ---- ci/clean_numba_cache.py | 85 +++++++++++++++++++++++++++++++------- docs/contributing.rst | 14 +++---- test/conftest.py | 32 ++++++++++++++ uxarray/grid/utils.py | 3 +- uxarray/utils/computing.py | 3 +- 6 files changed, 114 insertions(+), 32 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5cd15ce3a..c11e500cc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,12 +25,3 @@ repos: - id: ruff-format name: format with ruff types_or: [ python, pyi, jupyter ] - -- repo: local - hooks: - - id: clean-numba-cache - name: clear stale numba caches - entry: python ci/clean_numba_cache.py - language: python - files: ^uxarray/(utils/(computing|numba_math)|grid/(angles|arcs|area|coordinates|geometry|intersections|point_in_face|utils))\.py$ - pass_filenames: false diff --git a/ci/clean_numba_cache.py b/ci/clean_numba_cache.py index aa5daa647..383da3557 100644 --- a/ci/clean_numba_cache.py +++ b/ci/clean_numba_cache.py @@ -5,17 +5,43 @@ function from another module has that callee's code linked into its own cached object, so editing the callee leaves the caller's cache entry "valid" with the old body baked in. -Run this after editing any shared jitted module, or let the pre-commit hook do it:: +The test suite calls ``clear_if_shared_modules_changed`` at session start, which clears +the cache only when the jitted modules that others call into (``SHARED_MODULES``) differ +from the last run. Outside pytest (notebooks, scripts), run this by hand after editing +one of them:: python ci/clean_numba_cache.py [--dry-run] """ import argparse +import hashlib import os from pathlib import Path PACKAGE_DIR = Path(__file__).resolve().parent.parent / "uxarray" CACHE_SUFFIXES = {".nbi", ".nbc"} +# Jitted modules whose kernels are compiled into kernels of *other* modules. Add a module +# here when another module's jitted code starts calling into it. +SHARED_MODULES = [ + "utils/computing.py", + "utils/numba_math.py", + "grid/angles.py", + "grid/arcs.py", + "grid/area.py", + "grid/coordinates.py", + "grid/geometry.py", + "grid/intersections.py", + "grid/point_in_face.py", + "grid/utils.py", +] +# Digest of SHARED_MODULES as of the last clear; __pycache__ keeps it out of git. +STAMP = PACKAGE_DIR / "__pycache__" / "numba_shared_modules.sha256" +# Numba files caches in NUMBA_CACHE_DIR under a hash of each source directory, so they +# can't be told apart from other projects' entries; leave them alone rather than guess. +EXTERNAL_CACHE_WARNING = ( + "NUMBA_CACHE_DIR is set, so uxarray's numba cache lives there and was not cleared " + "(to avoid touching other projects' entries); clear it by hand." +) def _cache_files(): @@ -25,6 +51,47 @@ def _cache_files(): yield path +def _shared_modules_digest(): + digest = hashlib.sha256() + for name in SHARED_MODULES: + path = PACKAGE_DIR / name + digest.update(name.encode()) + digest.update(path.read_bytes() if path.exists() else b"") + return digest.hexdigest() + + +def _write_stamp(digest): + STAMP.parent.mkdir(exist_ok=True) + STAMP.write_text(digest) + + +def clear(dry_run=False): + """Delete every cache file; return how many there were.""" + files = sorted(set(_cache_files())) + for path in files: + if dry_run: + print(path) + else: + path.unlink(missing_ok=True) + + if not dry_run: + _write_stamp(_shared_modules_digest()) + return len(files) + + +def clear_if_shared_modules_changed(): + """Clear the cache if SHARED_MODULES changed since the last clear (or there is no + record of one); return the number of files removed, or None if nothing changed. + """ + digest = _shared_modules_digest() + try: + if STAMP.read_text() == digest: + return None + except OSError: + pass + return clear() + + def main(argv=None): parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument( @@ -35,21 +102,11 @@ def main(argv=None): ) args = parser.parse_args(argv) - files = sorted(set(_cache_files())) - for path in files: - if args.dry_run: - print(path) - else: - path.unlink(missing_ok=True) - + removed = clear(dry_run=args.dry_run) if os.environ.get("NUMBA_CACHE_DIR"): - # Numba files caches there under a hash of each source directory, so they cannot - # be told apart from other projects' entries; leave them alone rather than guess. - print("""WARNING: NUMBA_CACHE_DIR is set. To avoid corrupting caches of external projects, - no caches in this directory (NUMBA_CACHE_DIR) were deleted. Please clear it by hand""") - + print(f"WARNING: {EXTERNAL_CACHE_WARNING}") verb = "would remove" if args.dry_run else "removed" - print(f"{verb} {len(files)} numba cache file(s)") + print(f"{verb} {removed} numba cache file(s)") return 0 diff --git a/docs/contributing.rst b/docs/contributing.rst index a07d30b70..db4af0e6b 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -360,13 +360,13 @@ following command:: $ pre-commit run --all-files -One of the hooks, ``clean-numba-cache``, deletes numba's on-disk kernel cache when you stage a -change to one of the shared jitted modules (``uxarray/utils/computing.py``, ``numba_math.py`` -and the ``uxarray/grid`` modules that call them). Numba only checks a cached kernel against -the file that defines it, so a kernel that calls a jitted function from another file would -otherwise keep running the old code after that function is edited. The hook only fires at -commit time, so after editing one of these files, run the same cleanup by hand before you -test the change:: +A note on numba's kernel cache: numba only checks a cached kernel against the file that +defines it, so a kernel that calls a jitted function from another file keeps running the old +code after that function is edited. ``pytest`` handles this for you: at startup it clears the +cache if any of the shared jitted modules (``SHARED_MODULES`` in +``ci/clean_numba_cache.py``) differs from the last run, and leaves a warm cache alone +otherwise. Outside pytest (a notebook or script), clear it by hand after editing one of +those modules:: $ python ci/clean_numba_cache.py diff --git a/test/conftest.py b/test/conftest.py index 50ee2831b..e558e34ff 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,10 +1,42 @@ """Pytest configuration and fixtures for uxarray tests.""" +import importlib.util +import os + import pytest import numpy as np from pathlib import Path +@pytest.hookimpl(tryfirst=True) +def pytest_sessionstart(session): + """Clear numba's cache if a module other kernels compile in has changed. + + Runs before collection imports uxarray, so no stale kernel is loaded first. See + ci/clean_numba_cache.py for why numba's own staleness check misses this. + """ + if hasattr(session.config, "workerinput"): # pytest-xdist worker; controller did it + return + script = Path(__file__).parent.parent / "ci" / "clean_numba_cache.py" + if not script.exists(): # running from an installed package without the repo + return + spec = importlib.util.spec_from_file_location("clean_numba_cache", script) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + removed = module.clear_if_shared_modules_changed() + if removed is None: + return + reporter = session.config.pluginmanager.get_plugin("terminalreporter") + if reporter is None: + return + reporter.write_line( + f"uxarray: shared jitted modules changed; removed {removed} numba cache file(s)" + ) + if os.environ.get("NUMBA_CACHE_DIR"): + reporter.write_line(f"WARNING: {module.EXTERNAL_CACHE_WARNING}", yellow=True) + + @pytest.fixture def gridpath(): """Get the path to test grid/mesh file(s). diff --git a/uxarray/grid/utils.py b/uxarray/grid/utils.py index 320808cb1..56e849319 100644 --- a/uxarray/grid/utils.py +++ b/uxarray/grid/utils.py @@ -559,7 +559,8 @@ def setter(self, value): # # NOTE: these are inlined into ``cache=True`` kernels in another module, and numba stamps its # cache against the defining file alone, so editing them does not invalidate a caller's cached -# object. Run ``python ci/clean_numba_cache.py`` after changing anything here. +# object. pytest clears the cache when this file changes; outside pytest, run +# ``python ci/clean_numba_cache.py`` after changing anything here. @njit(cache=True) diff --git a/uxarray/utils/computing.py b/uxarray/utils/computing.py index 098c39ee5..a5e14f3ac 100644 --- a/uxarray/utils/computing.py +++ b/uxarray/utils/computing.py @@ -12,7 +12,8 @@ Every primitive here is inlined into ``cache=True`` kernels in other modules. Numba stamps a cache entry against the file that *defines* the kernel only, so after editing this file those callers keep running the old body from their cache, - with no warning. Run ``python ci/clean_numba_cache.py`` before testing changes here. + with no warning. pytest clears the cache when this file changes; elsewhere, run + ``python ci/clean_numba_cache.py`` before testing changes here. Python/Numba port of the AccuSphGeom C++ library (EFT tier only; the adaptive Shewchuk predicate and exact-arithmetic fallback tiers are not ported): From b6ba499460cb08a0759d79534ee7d3ae9f4b1c8f Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Wed, 18 Nov 2026 17:28:07 -0600 Subject: [PATCH 11/11] Test that SHARED_MODULES covers every cross-module kernel call The pytest-startup cache clear only watches SHARED_MODULES, a hand-kept list. test_shared_modules_complete walks every uxarray kernel's calls and fails, naming the module and its callers, if a module whose kernels are called from another module's kernels is missing, so the list can't silently fall behind. Co-Authored-By: Claude Opus 5.5 --- ci/clean_numba_cache.py | 3 ++- test/grid/geometry/test_numba_flags.py | 37 +++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/ci/clean_numba_cache.py b/ci/clean_numba_cache.py index 383da3557..4043131c8 100644 --- a/ci/clean_numba_cache.py +++ b/ci/clean_numba_cache.py @@ -21,7 +21,8 @@ PACKAGE_DIR = Path(__file__).resolve().parent.parent / "uxarray" CACHE_SUFFIXES = {".nbi", ".nbc"} # Jitted modules whose kernels are compiled into kernels of *other* modules. Add a module -# here when another module's jitted code starts calling into it. +# here when another module's jitted code starts calling into it; test_numba_flags.py's +# test_shared_modules_complete fails if one is missing. SHARED_MODULES = [ "utils/computing.py", "utils/numba_math.py", diff --git a/test/grid/geometry/test_numba_flags.py b/test/grid/geometry/test_numba_flags.py index 49245dda1..45a182eca 100644 --- a/test/grid/geometry/test_numba_flags.py +++ b/test/grid/geometry/test_numba_flags.py @@ -15,11 +15,14 @@ ``test_numpy_error_model`` checks every uxarray kernel that inlines one of ``NUMPY_ERROR_MODEL_KERNELS``; ``test_no_fastmath`` checks every kernel that inlines -anything from ``uxarray.utils.computing``. +anything from ``uxarray.utils.computing``. ``test_shared_modules_complete`` checks that +the cache clearing in test/conftest.py watches every module whose kernels are compiled +into another module's. """ import ast import importlib +import importlib.util import inspect import pkgutil import sys @@ -144,3 +147,35 @@ def test_no_fastmath(kernels): f"that isn't a literal False): they inline EFT code from {EFT_MODULE}, whose " f"error terms fastmath cancels to 0. Remove fastmath or set it to False." ) + + +def test_shared_modules_complete(kernels): + """SHARED_MODULES lists every module whose kernels another module's kernels call.""" + script = Path(__file__).parents[3] / "ci" / "clean_numba_cache.py" + if not script.exists(): + pytest.skip("ci/clean_numba_cache.py is not available outside the repository") + spec = importlib.util.spec_from_file_location("clean_numba_cache", script) + cleaner = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cleaner) + shared = { + "uxarray." + name.removesuffix(".py").replace("/", ".") + for name in cleaner.SHARED_MODULES + } + + # Any cross-module call counts, not only inline="always": numba links a cached + # callee's code into the caller's cached object either way. + called_from = {} + for kernel, _ in kernels.values(): + caller = kernel.py_func.__module__ + for callee in _callees(kernel): + if callee.py_func.__module__ != caller: + called_from.setdefault(callee.py_func.__module__, set()).add(caller) + assert called_from, "found no cross-module kernel calls; is _callees broken?" + + missing = {m: sorted(c) for m, c in sorted(called_from.items()) if m not in shared} + assert not missing, ( + f"Add these to SHARED_MODULES in ci/clean_numba_cache.py: {missing} " + f"(module -> its callers). Kernels in the callers compile in code from these " + f"modules, and numba won't notice when they change, so pytest must clear the " + f"cache whenever they do." + )