From d0dd85583e6d867b730e303c9ce97c3516d79177 Mon Sep 17 00:00:00 2001 From: Ziv Yaniv Date: Sun, 15 Nov 2026 10:13:04 -0500 Subject: [PATCH] Refactor notebook testing. Split the static (spellcheck, URL validity) and dynamic (notebook execution) testing so that we now have the option to run "static", "dynamic" or "both". Update the workflows so that the static checking is only performed once and the dynamic testing is performed on all platforms. Locally the default testing setup is "both". --- .circleci/config.yml | 32 ++++++++- .github/workflows/main.yml | 34 ++++++++- .github/workflows/scheduled_or_manual.yml | 21 +++++- tests/additional_dictionary.txt | 8 +++ tests/conftest.py | 19 +++++ tests/test_notebooks.py | 87 +++++++++++++++-------- 6 files changed, 166 insertions(+), 35 deletions(-) create mode 100644 tests/conftest.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 937bd075..d24dcc93 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,6 +38,27 @@ jobs: python -m pip install black[jupyter] python -m black --check --diff --verbose Python + static_check: + docker: + - image: cimg/python:3.13 + steps: + - checkout + - run: + name: Install dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements_dev.txt + - run: + name: static checking (spelling and more) + command: | + cd Python + NOTEBOOKS=$(ls *.ipynb) + cd .. + for notebook in $NOTEBOOKS + do + python -m pytest -v --tb=short --analysis=static tests/test_notebooks.py::Test_notebooks::test_python_notebook[$notebook] + done + test: parameters: os: @@ -87,7 +108,7 @@ jobs: cd .. for notebook in $TESTS do - python -m pytest -v --tb=short tests/test_notebooks.py::Test_notebooks::test_python_notebook[$notebook] + python -m pytest -v --tb=short --analysis=dynamic tests/test_notebooks.py::Test_notebooks::test_python_notebook[$notebook] done - save_cache: paths: @@ -98,13 +119,20 @@ workflows: when: << pipeline.parameters.run_monthly_testing >> jobs: - lint + - static_check: + requires: + - lint + filters: + branches: + ignore: + - '**.md' - test: matrix: parameters: python-version: ['3.10', '3.13'] os: ["linux"] requires: - - lint + - static_check filters: branches: ignore: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8e2515af..9574c716 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -30,8 +30,38 @@ jobs: python -m pip install --upgrade pip python -m pip install black[jupyter] black --check --diff --verbose Python - test: + static_check: + name: static checking (spelling and more) needs: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 # fetch all history for all branches, required for the changed-files action + - name: Set up Python 3.13 + uses: actions/setup-python@v7 + with: + python-version: '3.13' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements_dev.txt + - name: Get list of Python notebooks to test + id: nbl + uses: tj-actions/changed-files@v47 + with: + files: Python/*.ipynb + - name: Run static analysis on the notebooks + shell: bash + if: steps.nbl.outputs.any_changed == 'true' + run: | + test_files="${{ steps.nbl.outputs.all_changed_files }}" + for nb_file in ${test_files}; do + # the substring contains the file paths, extract the basename for usage with pytest + pytest -v --tb=short --analysis=static tests/test_notebooks.py::Test_notebooks::test_python_notebook["${nb_file##*/}"] + done + test: + needs: static_check strategy: matrix: os: [windows-latest, ubuntu-latest, macos-latest] @@ -78,7 +108,7 @@ jobs: fi for nb_file in ${test_files}; do # the substring contains the file paths, extract the basename for usage with pytest - pytest -v --tb=short tests/test_notebooks.py::Test_notebooks::test_python_notebook["${nb_file##*/}"] + pytest -v --tb=short --analysis=dynamic tests/test_notebooks.py::Test_notebooks::test_python_notebook["${nb_file##*/}"] done - name: Run the scripts test env: diff --git a/.github/workflows/scheduled_or_manual.yml b/.github/workflows/scheduled_or_manual.yml index 81d2dae8..e5477e8d 100644 --- a/.github/workflows/scheduled_or_manual.yml +++ b/.github/workflows/scheduled_or_manual.yml @@ -31,8 +31,25 @@ jobs: python -m pip install --upgrade pip python -m pip install black[jupyter] black --check --diff --verbose Python - test: + static_check: + name: static checking (spelling and more) needs: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Set up Python 3.13 + uses: actions/setup-python@v7 + with: + python-version: '3.13' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements_dev.txt + - name: run static analysis on all notebooks + run: | + pytest -v --tb=short --analysis=static tests/test_notebooks.py::Test_notebooks::test_python_notebook + test: + needs: static_check strategy: matrix: os: [windows-latest, macos-latest] @@ -68,5 +85,5 @@ jobs: env: SIMPLE_ITK_MEMORY_CONSTRAINED_ENVIRONMENT: 1 run: | - pytest -v --tb=short -k "${{matrix.inputs}}" tests/test_notebooks.py::Test_notebooks::test_python_notebook + pytest -v --tb=short --analysis=dynamic -k "${{matrix.inputs}}" tests/test_notebooks.py::Test_notebooks::test_python_notebook pytest -v --tb=short tests/test_scripts.py diff --git a/tests/additional_dictionary.txt b/tests/additional_dictionary.txt index 54578d9b..1d2408b4 100644 --- a/tests/additional_dictionary.txt +++ b/tests/additional_dictionary.txt @@ -1,4 +1,12 @@ streamable +namespace +uuid +divid +enum +enums +Gabor +functionalities +std rm Assoc phys diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..d27d36d4 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,19 @@ +# Custom pytest CLI options must be registered via pytest_addoption in a +# conftest.py (pytest parses arguments before test modules are imported, so +# this hook cannot live in test_notebooks.py itself). This lets CI split +# notebook testing into separate static/dynamic steps while local runs +# default to doing both. +def pytest_addoption(parser): + parser.addoption( + "--analysis", + action="store", + default="both", + choices=["static", "dynamic", "both"], + help=( + "Which notebook analysis to run in " + "Test_notebooks::test_python_notebook/test_r_notebook: " + "'static' (no output/broken-links/spelling checks), " + "'dynamic' (execute notebook, check for errors), " + "or 'both' (default, used for local testing)." + ), + ) diff --git a/tests/test_notebooks.py b/tests/test_notebooks.py index e231ab57..b9f65346 100755 --- a/tests/test_notebooks.py +++ b/tests/test_notebooks.py @@ -32,6 +32,16 @@ run specific R test: pytest -v --tb=short tests/test_notebooks.py::Test_notebooks::test_r_notebook[00_Setup.ipynb] +By default both static and dynamic analysis are performed (appropriate for +local testing). On CI, the two are split into separate jobs/steps, each +running once, via the --analysis option which is defined via the conftest.py: + +run only the static analysis (no output/broken-links/spelling checks): +pytest -v --tb=short --analysis=static tests/test_notebooks.py::Test_notebooks::test_python_notebook + +run only the dynamic analysis (execute notebook, check for errors): +pytest -v --tb=short --analysis=dynamic tests/test_notebooks.py::Test_notebooks::test_python_notebook + -s : disable all capturing of output. """ @@ -76,19 +86,31 @@ class Test_notebooks(object): _allowed_error_markup = "simpleitk_error_allowed" _expected_error_markup = "simpleitk_error_expected" + # Notebooks that are skipped when dynamic analysis runs (dynamic or both + # modes), keyed by file name, value is (skip_condition, reason). Static-only + # runs ignore these and always test the notebook. + _dynamic_skip_conditions = { + "03_Image_Details.ipynb": ( + os.environ.get("GITHUB_ACTIONS") == "true", + "on GitHub runners, nbconvert intermittently fails with dead kernel, even after reducing notebook memory usage", + ), + "35_Segmentation_Shape_Analysis.ipynb": ( + os.environ.get("CIRCLECI") == "true", + "runtime too long for CircleCI", + ), + "11_Progress.ipynb": ( + True, + "this notebook times out when run with nbconvert, due to javascript issues, so not tested", + ), + } + @pytest.mark.parametrize( "notebook_file_name", [ "00_Setup.ipynb", "01_Image_Basics.ipynb", "02_Pythonic_Image.ipynb", - pytest.param( - "03_Image_Details.ipynb", - marks=pytest.mark.skipif( - os.environ.get("GITHUB_ACTIONS") == "true", - reason="on GitHub runners, nbconvert intermittently fails with dead kernel, even after reducing notebook memory usage", - ), - ), + "03_Image_Details.ipynb", "04_Image_Display.ipynb", "05_Results_Visualization.ipynb", "10_matplotlibs_imshow.ipynb", @@ -101,21 +123,9 @@ class Test_notebooks(object): "32_Watersheds_Segmentation.ipynb", "33_Segmentation_Thresholding_Edge_Detection.ipynb", "34_Segmentation_Evaluation.ipynb", - pytest.param( - "35_Segmentation_Shape_Analysis.ipynb", - marks=pytest.mark.skipif( - os.environ.get("CIRCLECI") == "true", - reason="runtime too long for CircleCI", - ), - ), + "35_Segmentation_Shape_Analysis.ipynb", "36_Microscopy_Colocalization_Distance_Analysis.ipynb", - pytest.param( - "11_Progress.ipynb", - marks=pytest.mark.skipif( - True, - reason="this notebook times out when run with nbconvert, due to javascript issues, so not tested", - ), - ), + "11_Progress.ipynb", "51_VH_Segmentation1.ipynb", "55_VH_Resample.ipynb", "56_VH_Registration1.ipynb", @@ -133,8 +143,17 @@ class Test_notebooks(object): "71_Trust_But_Verify.ipynb", ], ) - def test_python_notebook(self, notebook_file_name): - self.evaluate_notebook(self.absolute_path_python(notebook_file_name), "python") + def test_python_notebook(self, notebook_file_name, request): + analysis = request.config.getoption("--analysis") + if analysis != "static": + skip_condition = self._dynamic_skip_conditions.get(notebook_file_name) + if skip_condition and skip_condition[0]: + pytest.skip(skip_condition[1]) + self.evaluate_notebook( + self.absolute_path_python(notebook_file_name), + "python", + analysis, + ) @pytest.mark.parametrize( "notebook_file_name", @@ -154,17 +173,23 @@ def test_python_notebook(self, notebook_file_name): "70_Data_Augmentation.ipynb", ], ) - def test_r_notebook(self, notebook_file_name): - self.evaluate_notebook(self.absolute_path_r(notebook_file_name), "ir") + def test_r_notebook(self, notebook_file_name, request): + self.evaluate_notebook( + self.absolute_path_r(notebook_file_name), + "ir", + request.config.getoption("--analysis"), + ) - def evaluate_notebook(self, path, kernel_name): + def evaluate_notebook(self, path, kernel_name, analysis="both"): """ - Perform static and dynamic analysis of the notebook. + Perform static and/or dynamic analysis of the notebook. Execute a notebook via nbconvert and print the results of the test (errors etc.) Args: path (string): Name of notebook to run. kernel_name (string): Which jupyter kernel to use to run the test. Relevant values are:'python2', 'python3', 'ir'. + analysis (string): Which analysis to perform, one of 'static', + 'dynamic', or 'both' (default). """ dir_name, file_name = os.path.split(path) @@ -172,8 +197,12 @@ def evaluate_notebook(self, path, kernel_name): os.chdir(dir_name) print("-------- begin (kernel {0}) {1} --------".format(kernel_name, file_name)) - no_static_errors = self.static_analysis(path) - no_dynamic_errors = self.dynamic_analysis(path, kernel_name) + no_static_errors = True + no_dynamic_errors = True + if analysis in ("static", "both"): + no_static_errors = self.static_analysis(path) + if analysis in ("dynamic", "both"): + no_dynamic_errors = self.dynamic_analysis(path, kernel_name) print("-------- end (kernel {0}) {1} --------".format(kernel_name, file_name)) assert no_static_errors and no_dynamic_errors