Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 30 additions & 2 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,27 @@ jobs:
python -m pip install black[jupyter]
python -m black --check --diff --verbose Python

static_check:
docker:
- image: cimg/python:3.13
steps:
- checkout
- run:
name: Install dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r requirements_dev.txt
- run:
name: static checking (spelling and more)
command: |
cd Python
NOTEBOOKS=$(ls *.ipynb)
cd ..
for notebook in $NOTEBOOKS
do
python -m pytest -v --tb=short --analysis=static tests/test_notebooks.py::Test_notebooks::test_python_notebook[$notebook]
done

test:
parameters:
os:
Expand Down Expand Up @@ -87,7 +108,7 @@ jobs:
cd ..
for notebook in $TESTS
do
python -m pytest -v --tb=short tests/test_notebooks.py::Test_notebooks::test_python_notebook[$notebook]
python -m pytest -v --tb=short --analysis=dynamic tests/test_notebooks.py::Test_notebooks::test_python_notebook[$notebook]
done
- save_cache:
paths:
Expand All @@ -98,13 +119,20 @@ workflows:
when: << pipeline.parameters.run_monthly_testing >>
jobs:
- lint
- static_check:
requires:
- lint
filters:
branches:
ignore:
- '**.md'
- test:
matrix:
parameters:
python-version: ['3.10', '3.13']
os: ["linux"]
requires:
- lint
- static_check
filters:
branches:
ignore:
Expand Down
34 changes: 32 additions & 2 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,38 @@ jobs:
python -m pip install --upgrade pip
python -m pip install black[jupyter]
black --check --diff --verbose Python
test:
static_check:
name: static checking (spelling and more)
needs: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
with:
fetch-depth: 0 # fetch all history for all branches, required for the changed-files action
- name: Set up Python 3.13
uses: actions/setup-python@v7
with:
python-version: '3.13'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
python -m pip install -r requirements_dev.txt
- name: Get list of Python notebooks to test
id: nbl
uses: tj-actions/changed-files@v47
with:
files: Python/*.ipynb
- name: Run static analysis on the notebooks
shell: bash
if: steps.nbl.outputs.any_changed == 'true'
run: |
test_files="${{ steps.nbl.outputs.all_changed_files }}"
for nb_file in ${test_files}; do
# the substring contains the file paths, extract the basename for usage with pytest
pytest -v --tb=short --analysis=static tests/test_notebooks.py::Test_notebooks::test_python_notebook["${nb_file##*/}"]
done
test:
needs: static_check
strategy:
matrix:
os: [windows-latest, ubuntu-latest, macos-latest]
Expand Down Expand Up @@ -78,7 +108,7 @@ jobs:
fi
for nb_file in ${test_files}; do
# the substring contains the file paths, extract the basename for usage with pytest
pytest -v --tb=short tests/test_notebooks.py::Test_notebooks::test_python_notebook["${nb_file##*/}"]
pytest -v --tb=short --analysis=dynamic tests/test_notebooks.py::Test_notebooks::test_python_notebook["${nb_file##*/}"]
done
- name: Run the scripts test
env:
Expand Down
21 changes: 19 additions & 2 deletions .github/workflows/scheduled_or_manual.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,25 @@ jobs:
python -m pip install --upgrade pip
python -m pip install black[jupyter]
black --check --diff --verbose Python
test:
static_check:
name: static checking (spelling and more)
needs: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- name: Set up Python 3.13
uses: actions/setup-python@v7
with:
python-version: '3.13'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
python -m pip install -r requirements_dev.txt
- name: run static analysis on all notebooks
run: |
pytest -v --tb=short --analysis=static tests/test_notebooks.py::Test_notebooks::test_python_notebook
test:
needs: static_check
strategy:
matrix:
os: [windows-latest, macos-latest]
Expand Down Expand Up @@ -68,5 +85,5 @@ jobs:
env:
SIMPLE_ITK_MEMORY_CONSTRAINED_ENVIRONMENT: 1
run: |
pytest -v --tb=short -k "${{matrix.inputs}}" tests/test_notebooks.py::Test_notebooks::test_python_notebook
pytest -v --tb=short --analysis=dynamic -k "${{matrix.inputs}}" tests/test_notebooks.py::Test_notebooks::test_python_notebook
pytest -v --tb=short tests/test_scripts.py
8 changes: 8 additions & 0 deletions tests/additional_dictionary.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,12 @@
streamable
namespace
uuid
divid
enum
enums
Gabor
functionalities
std
rm
Assoc
phys
Expand Down
19 changes: 19 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Custom pytest CLI options must be registered via pytest_addoption in a
# conftest.py (pytest parses arguments before test modules are imported, so
# this hook cannot live in test_notebooks.py itself). This lets CI split
# notebook testing into separate static/dynamic steps while local runs
# default to doing both.
def pytest_addoption(parser):
parser.addoption(
"--analysis",
action="store",
default="both",
choices=["static", "dynamic", "both"],
help=(
"Which notebook analysis to run in "
"Test_notebooks::test_python_notebook/test_r_notebook: "
"'static' (no output/broken-links/spelling checks), "
"'dynamic' (execute notebook, check for errors), "
"or 'both' (default, used for local testing)."
),
)
87 changes: 58 additions & 29 deletions tests/test_notebooks.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,16 @@
run specific R test:
pytest -v --tb=short tests/test_notebooks.py::Test_notebooks::test_r_notebook[00_Setup.ipynb]

By default both static and dynamic analysis are performed (appropriate for
local testing). On CI, the two are split into separate jobs/steps, each
running once, via the --analysis option which is defined via the conftest.py:

run only the static analysis (no output/broken-links/spelling checks):
pytest -v --tb=short --analysis=static tests/test_notebooks.py::Test_notebooks::test_python_notebook

run only the dynamic analysis (execute notebook, check for errors):
pytest -v --tb=short --analysis=dynamic tests/test_notebooks.py::Test_notebooks::test_python_notebook

-s : disable all capturing of output.
"""

Expand Down Expand Up @@ -76,19 +86,31 @@ class Test_notebooks(object):
_allowed_error_markup = "simpleitk_error_allowed"
_expected_error_markup = "simpleitk_error_expected"

# Notebooks that are skipped when dynamic analysis runs (dynamic or both
# modes), keyed by file name, value is (skip_condition, reason). Static-only
# runs ignore these and always test the notebook.
_dynamic_skip_conditions = {
"03_Image_Details.ipynb": (
os.environ.get("GITHUB_ACTIONS") == "true",
"on GitHub runners, nbconvert intermittently fails with dead kernel, even after reducing notebook memory usage",
),
"35_Segmentation_Shape_Analysis.ipynb": (
os.environ.get("CIRCLECI") == "true",
"runtime too long for CircleCI",
),
"11_Progress.ipynb": (
True,
"this notebook times out when run with nbconvert, due to javascript issues, so not tested",
),
}

@pytest.mark.parametrize(
"notebook_file_name",
[
"00_Setup.ipynb",
"01_Image_Basics.ipynb",
"02_Pythonic_Image.ipynb",
pytest.param(
"03_Image_Details.ipynb",
marks=pytest.mark.skipif(
os.environ.get("GITHUB_ACTIONS") == "true",
reason="on GitHub runners, nbconvert intermittently fails with dead kernel, even after reducing notebook memory usage",
),
),
"03_Image_Details.ipynb",
"04_Image_Display.ipynb",
"05_Results_Visualization.ipynb",
"10_matplotlibs_imshow.ipynb",
Expand All @@ -101,21 +123,9 @@ class Test_notebooks(object):
"32_Watersheds_Segmentation.ipynb",
"33_Segmentation_Thresholding_Edge_Detection.ipynb",
"34_Segmentation_Evaluation.ipynb",
pytest.param(
"35_Segmentation_Shape_Analysis.ipynb",
marks=pytest.mark.skipif(
os.environ.get("CIRCLECI") == "true",
reason="runtime too long for CircleCI",
),
),
"35_Segmentation_Shape_Analysis.ipynb",
"36_Microscopy_Colocalization_Distance_Analysis.ipynb",
pytest.param(
"11_Progress.ipynb",
marks=pytest.mark.skipif(
True,
reason="this notebook times out when run with nbconvert, due to javascript issues, so not tested",
),
),
"11_Progress.ipynb",
"51_VH_Segmentation1.ipynb",
"55_VH_Resample.ipynb",
"56_VH_Registration1.ipynb",
Expand All @@ -133,8 +143,17 @@ class Test_notebooks(object):
"71_Trust_But_Verify.ipynb",
],
)
def test_python_notebook(self, notebook_file_name):
self.evaluate_notebook(self.absolute_path_python(notebook_file_name), "python")
def test_python_notebook(self, notebook_file_name, request):
analysis = request.config.getoption("--analysis")
if analysis != "static":
skip_condition = self._dynamic_skip_conditions.get(notebook_file_name)
if skip_condition and skip_condition[0]:
pytest.skip(skip_condition[1])
self.evaluate_notebook(
self.absolute_path_python(notebook_file_name),
"python",
analysis,
)

@pytest.mark.parametrize(
"notebook_file_name",
Expand All @@ -154,26 +173,36 @@ def test_python_notebook(self, notebook_file_name):
"70_Data_Augmentation.ipynb",
],
)
def test_r_notebook(self, notebook_file_name):
self.evaluate_notebook(self.absolute_path_r(notebook_file_name), "ir")
def test_r_notebook(self, notebook_file_name, request):
self.evaluate_notebook(
self.absolute_path_r(notebook_file_name),
"ir",
request.config.getoption("--analysis"),
)

def evaluate_notebook(self, path, kernel_name):
def evaluate_notebook(self, path, kernel_name, analysis="both"):
"""
Perform static and dynamic analysis of the notebook.
Perform static and/or dynamic analysis of the notebook.
Execute a notebook via nbconvert and print the results of the test (errors etc.)
Args:
path (string): Name of notebook to run.
kernel_name (string): Which jupyter kernel to use to run the test.
Relevant values are:'python2', 'python3', 'ir'.
analysis (string): Which analysis to perform, one of 'static',
'dynamic', or 'both' (default).
"""

dir_name, file_name = os.path.split(path)
if dir_name:
os.chdir(dir_name)

print("-------- begin (kernel {0}) {1} --------".format(kernel_name, file_name))
no_static_errors = self.static_analysis(path)
no_dynamic_errors = self.dynamic_analysis(path, kernel_name)
no_static_errors = True
no_dynamic_errors = True
if analysis in ("static", "both"):
no_static_errors = self.static_analysis(path)
if analysis in ("dynamic", "both"):
no_dynamic_errors = self.dynamic_analysis(path, kernel_name)
print("-------- end (kernel {0}) {1} --------".format(kernel_name, file_name))
assert no_static_errors and no_dynamic_errors

Expand Down
Loading