diff --git a/.circleci/config.yml b/.circleci/config.yml index 937bd075..d24dcc93 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,6 +38,27 @@ jobs: python -m pip install black[jupyter] python -m black --check --diff --verbose Python + static_check: + docker: + - image: cimg/python:3.13 + steps: + - checkout + - run: + name: Install dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements_dev.txt + - run: + name: static checking (spelling and more) + command: | + cd Python + NOTEBOOKS=$(ls *.ipynb) + cd .. + for notebook in $NOTEBOOKS + do + python -m pytest -v --tb=short --analysis=static tests/test_notebooks.py::Test_notebooks::test_python_notebook[$notebook] + done + test: parameters: os: @@ -87,7 +108,7 @@ jobs: cd .. for notebook in $TESTS do - python -m pytest -v --tb=short tests/test_notebooks.py::Test_notebooks::test_python_notebook[$notebook] + python -m pytest -v --tb=short --analysis=dynamic tests/test_notebooks.py::Test_notebooks::test_python_notebook[$notebook] done - save_cache: paths: @@ -98,13 +119,20 @@ workflows: when: << pipeline.parameters.run_monthly_testing >> jobs: - lint + - static_check: + requires: + - lint + filters: + branches: + ignore: + - '**.md' - test: matrix: parameters: python-version: ['3.10', '3.13'] os: ["linux"] requires: - - lint + - static_check filters: branches: ignore: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8e2515af..9574c716 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -30,8 +30,38 @@ jobs: python -m pip install --upgrade pip python -m pip install black[jupyter] black --check --diff --verbose Python - test: + static_check: + name: static checking (spelling and more) needs: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 # fetch all history for all branches, required for the changed-files action + - name: Set up Python 3.13 + uses: actions/setup-python@v7 + with: + python-version: '3.13' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements_dev.txt + - name: Get list of Python notebooks to test + id: nbl + uses: tj-actions/changed-files@v47 + with: + files: Python/*.ipynb + - name: Run static analysis on the notebooks + shell: bash + if: steps.nbl.outputs.any_changed == 'true' + run: | + test_files="${{ steps.nbl.outputs.all_changed_files }}" + for nb_file in ${test_files}; do + # the substring contains the file paths, extract the basename for usage with pytest + pytest -v --tb=short --analysis=static tests/test_notebooks.py::Test_notebooks::test_python_notebook["${nb_file##*/}"] + done + test: + needs: static_check strategy: matrix: os: [windows-latest, ubuntu-latest, macos-latest] @@ -78,7 +108,7 @@ jobs: fi for nb_file in ${test_files}; do # the substring contains the file paths, extract the basename for usage with pytest - pytest -v --tb=short tests/test_notebooks.py::Test_notebooks::test_python_notebook["${nb_file##*/}"] + pytest -v --tb=short --analysis=dynamic tests/test_notebooks.py::Test_notebooks::test_python_notebook["${nb_file##*/}"] done - name: Run the scripts test env: diff --git a/.github/workflows/scheduled_or_manual.yml b/.github/workflows/scheduled_or_manual.yml index 81d2dae8..e5477e8d 100644 --- a/.github/workflows/scheduled_or_manual.yml +++ b/.github/workflows/scheduled_or_manual.yml @@ -31,8 +31,25 @@ jobs: python -m pip install --upgrade pip python -m pip install black[jupyter] black --check --diff --verbose Python - test: + static_check: + name: static checking (spelling and more) needs: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Set up Python 3.13 + uses: actions/setup-python@v7 + with: + python-version: '3.13' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements_dev.txt + - name: run static analysis on all notebooks + run: | + pytest -v --tb=short --analysis=static tests/test_notebooks.py::Test_notebooks::test_python_notebook + test: + needs: static_check strategy: matrix: os: [windows-latest, macos-latest] @@ -68,5 +85,5 @@ jobs: env: SIMPLE_ITK_MEMORY_CONSTRAINED_ENVIRONMENT: 1 run: | - pytest -v --tb=short -k "${{matrix.inputs}}" tests/test_notebooks.py::Test_notebooks::test_python_notebook + pytest -v --tb=short --analysis=dynamic -k "${{matrix.inputs}}" tests/test_notebooks.py::Test_notebooks::test_python_notebook pytest -v --tb=short tests/test_scripts.py diff --git a/tests/additional_dictionary.txt b/tests/additional_dictionary.txt index 54578d9b..1d2408b4 100644 --- a/tests/additional_dictionary.txt +++ b/tests/additional_dictionary.txt @@ -1,4 +1,12 @@ streamable +namespace +uuid +divid +enum +enums +Gabor +functionalities +std rm Assoc phys diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..d27d36d4 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,19 @@ +# Custom pytest CLI options must be registered via pytest_addoption in a +# conftest.py (pytest parses arguments before test modules are imported, so +# this hook cannot live in test_notebooks.py itself). This lets CI split +# notebook testing into separate static/dynamic steps while local runs +# default to doing both. +def pytest_addoption(parser): + parser.addoption( + "--analysis", + action="store", + default="both", + choices=["static", "dynamic", "both"], + help=( + "Which notebook analysis to run in " + "Test_notebooks::test_python_notebook/test_r_notebook: " + "'static' (no output/broken-links/spelling checks), " + "'dynamic' (execute notebook, check for errors), " + "or 'both' (default, used for local testing)." + ), + ) diff --git a/tests/test_notebooks.py b/tests/test_notebooks.py index e231ab57..b9f65346 100755 --- a/tests/test_notebooks.py +++ b/tests/test_notebooks.py @@ -32,6 +32,16 @@ run specific R test: pytest -v --tb=short tests/test_notebooks.py::Test_notebooks::test_r_notebook[00_Setup.ipynb] +By default both static and dynamic analysis are performed (appropriate for +local testing). On CI, the two are split into separate jobs/steps, each +running once, via the --analysis option which is defined via the conftest.py: + +run only the static analysis (no output/broken-links/spelling checks): +pytest -v --tb=short --analysis=static tests/test_notebooks.py::Test_notebooks::test_python_notebook + +run only the dynamic analysis (execute notebook, check for errors): +pytest -v --tb=short --analysis=dynamic tests/test_notebooks.py::Test_notebooks::test_python_notebook + -s : disable all capturing of output. """ @@ -76,19 +86,31 @@ class Test_notebooks(object): _allowed_error_markup = "simpleitk_error_allowed" _expected_error_markup = "simpleitk_error_expected" + # Notebooks that are skipped when dynamic analysis runs (dynamic or both + # modes), keyed by file name, value is (skip_condition, reason). Static-only + # runs ignore these and always test the notebook. + _dynamic_skip_conditions = { + "03_Image_Details.ipynb": ( + os.environ.get("GITHUB_ACTIONS") == "true", + "on GitHub runners, nbconvert intermittently fails with dead kernel, even after reducing notebook memory usage", + ), + "35_Segmentation_Shape_Analysis.ipynb": ( + os.environ.get("CIRCLECI") == "true", + "runtime too long for CircleCI", + ), + "11_Progress.ipynb": ( + True, + "this notebook times out when run with nbconvert, due to javascript issues, so not tested", + ), + } + @pytest.mark.parametrize( "notebook_file_name", [ "00_Setup.ipynb", "01_Image_Basics.ipynb", "02_Pythonic_Image.ipynb", - pytest.param( - "03_Image_Details.ipynb", - marks=pytest.mark.skipif( - os.environ.get("GITHUB_ACTIONS") == "true", - reason="on GitHub runners, nbconvert intermittently fails with dead kernel, even after reducing notebook memory usage", - ), - ), + "03_Image_Details.ipynb", "04_Image_Display.ipynb", "05_Results_Visualization.ipynb", "10_matplotlibs_imshow.ipynb", @@ -101,21 +123,9 @@ class Test_notebooks(object): "32_Watersheds_Segmentation.ipynb", "33_Segmentation_Thresholding_Edge_Detection.ipynb", "34_Segmentation_Evaluation.ipynb", - pytest.param( - "35_Segmentation_Shape_Analysis.ipynb", - marks=pytest.mark.skipif( - os.environ.get("CIRCLECI") == "true", - reason="runtime too long for CircleCI", - ), - ), + "35_Segmentation_Shape_Analysis.ipynb", "36_Microscopy_Colocalization_Distance_Analysis.ipynb", - pytest.param( - "11_Progress.ipynb", - marks=pytest.mark.skipif( - True, - reason="this notebook times out when run with nbconvert, due to javascript issues, so not tested", - ), - ), + "11_Progress.ipynb", "51_VH_Segmentation1.ipynb", "55_VH_Resample.ipynb", "56_VH_Registration1.ipynb", @@ -133,8 +143,17 @@ class Test_notebooks(object): "71_Trust_But_Verify.ipynb", ], ) - def test_python_notebook(self, notebook_file_name): - self.evaluate_notebook(self.absolute_path_python(notebook_file_name), "python") + def test_python_notebook(self, notebook_file_name, request): + analysis = request.config.getoption("--analysis") + if analysis != "static": + skip_condition = self._dynamic_skip_conditions.get(notebook_file_name) + if skip_condition and skip_condition[0]: + pytest.skip(skip_condition[1]) + self.evaluate_notebook( + self.absolute_path_python(notebook_file_name), + "python", + analysis, + ) @pytest.mark.parametrize( "notebook_file_name", @@ -154,17 +173,23 @@ def test_python_notebook(self, notebook_file_name): "70_Data_Augmentation.ipynb", ], ) - def test_r_notebook(self, notebook_file_name): - self.evaluate_notebook(self.absolute_path_r(notebook_file_name), "ir") + def test_r_notebook(self, notebook_file_name, request): + self.evaluate_notebook( + self.absolute_path_r(notebook_file_name), + "ir", + request.config.getoption("--analysis"), + ) - def evaluate_notebook(self, path, kernel_name): + def evaluate_notebook(self, path, kernel_name, analysis="both"): """ - Perform static and dynamic analysis of the notebook. + Perform static and/or dynamic analysis of the notebook. Execute a notebook via nbconvert and print the results of the test (errors etc.) Args: path (string): Name of notebook to run. kernel_name (string): Which jupyter kernel to use to run the test. Relevant values are:'python2', 'python3', 'ir'. + analysis (string): Which analysis to perform, one of 'static', + 'dynamic', or 'both' (default). """ dir_name, file_name = os.path.split(path) @@ -172,8 +197,12 @@ def evaluate_notebook(self, path, kernel_name): os.chdir(dir_name) print("-------- begin (kernel {0}) {1} --------".format(kernel_name, file_name)) - no_static_errors = self.static_analysis(path) - no_dynamic_errors = self.dynamic_analysis(path, kernel_name) + no_static_errors = True + no_dynamic_errors = True + if analysis in ("static", "both"): + no_static_errors = self.static_analysis(path) + if analysis in ("dynamic", "both"): + no_dynamic_errors = self.dynamic_analysis(path, kernel_name) print("-------- end (kernel {0}) {1} --------".format(kernel_name, file_name)) assert no_static_errors and no_dynamic_errors