From 151a1d1ca34ce1f80b1c5621f129b85a7ff87401 Mon Sep 17 00:00:00 2001 From: Nakul Santhosh Date: Wed, 10 Jun 2026 10:38:07 +0200 Subject: [PATCH 1/4] feat: initial notebook test script --- pyproject.toml | 1 + tests/test_notebook.py | 64 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100755 tests/test_notebook.py diff --git a/pyproject.toml b/pyproject.toml index 4a61c1d4..0eec468f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,6 +129,7 @@ markers = [ "unit: unit tests", "json_files: Export JSON files with --update-json-files", "out_files: Export .out fixture files with --update-out-files", + "notebooks: tests that run Jupyter notebooks", ] diff --git a/tests/test_notebook.py b/tests/test_notebook.py new file mode 100755 index 00000000..0b3d12b6 --- /dev/null +++ b/tests/test_notebook.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +"""Run all notebooks in tests/notebooks/ and report pass/fail.""" + +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +import pytest + +# NOTEBOOKS_DIR = Path(__file__).parent.parent / "docs/contents/notebooks" +NOTEBOOKS_DIR = Path(__file__).parent / "notebooks" + + +def run_notebook(nb: Path) -> tuple[bool, float, str]: + start = time.monotonic() + with tempfile.TemporaryDirectory() as tmp: + result = subprocess.run( + [ + "jupyter", + "nbconvert", + "--execute", + "--to", + "notebook", + "--ExecutePreprocessor.timeout=1800", + f"--ExecutePreprocessor.kernel_cwd={tmp}", + "--output", + nb.name, + str(nb), + ], + capture_output=True, + text=True, + cwd=nb.parent, + ) + elapsed = time.monotonic() - start + return result.returncode == 0, elapsed, result.stdout + result.stderr + + +@pytest.mark.notebooks +def test_notebooks() -> None: + notebooks = sorted(NOTEBOOKS_DIR.glob("*.ipynb")) + if not notebooks: + print(f"No notebooks found in {NOTEBOOKS_DIR}") + sys.exit(1) + + results: list[tuple[str, bool, float]] = [] + for nb in notebooks: + print(f" running {nb.name} ...", end="", flush=True) + ok, elapsed, output = run_notebook(nb) + status = "PASS" if ok else "FAIL" + print(f" {status} ({elapsed:.1f}s)") + if not ok: + print(output) + results.append((nb.name, ok, elapsed)) + + print() + failures = [name for name, ok, _ in results if not ok] + print(f"{len(notebooks) - len(failures)}/{len(notebooks)} notebooks passed") + if failures: + print("Failed:") + for name in failures: + print(f" - {name}") + sys.exit(1) From 8b80f80b96594cba26fe3444b0219d69ea7c50ad Mon Sep 17 00:00:00 2001 From: Nakul Santhosh Date: Wed, 24 Jun 2026 13:37:46 +0200 Subject: [PATCH 2/4] fix: fix the NOTEBOOKS_DIR path and add temp directory as working directory for notebook tests --- tests/test_notebook.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_notebook.py b/tests/test_notebook.py index 0b3d12b6..e992a900 100755 --- a/tests/test_notebook.py +++ b/tests/test_notebook.py @@ -9,8 +9,7 @@ import pytest -# NOTEBOOKS_DIR = Path(__file__).parent.parent / "docs/contents/notebooks" -NOTEBOOKS_DIR = Path(__file__).parent / "notebooks" +NOTEBOOKS_DIR = Path(__file__).parent.parent / "docs/contents/notebooks" def run_notebook(nb: Path) -> tuple[bool, float, str]: @@ -27,6 +26,8 @@ def run_notebook(nb: Path) -> tuple[bool, float, str]: f"--ExecutePreprocessor.kernel_cwd={tmp}", "--output", nb.name, + "--output-dir", + tmp, str(nb), ], capture_output=True, From d78e11b579a701909f46411f96bb7c7c15e4e79f Mon Sep 17 00:00:00 2001 From: Nakul Santhosh Date: Wed, 1 Jul 2026 07:54:10 +0200 Subject: [PATCH 3/4] fix: add BLACKLIST and fix bug of notebooks creating output directories in notebook directory --- tests/test_notebook.py | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/tests/test_notebook.py b/tests/test_notebook.py index e992a900..89957f43 100755 --- a/tests/test_notebook.py +++ b/tests/test_notebook.py @@ -1,6 +1,7 @@ #!/usr/bin/env python """Run all notebooks in tests/notebooks/ and report pass/fail.""" +import shutil import subprocess import sys import tempfile @@ -10,11 +11,28 @@ import pytest NOTEBOOKS_DIR = Path(__file__).parent.parent / "docs/contents/notebooks" +BLACKLIST = [ + "extopt", + "ir_spectrum", + "ml_properties" +] def run_notebook(nb: Path) -> tuple[bool, float, str]: + """ + Helper function that runs a specific notebook and reports pass/fail. + Parameters + ---------- + nb + + Returns + ------- + + """ start = time.monotonic() with tempfile.TemporaryDirectory() as tmp: + nb_copy = Path(tmp) / nb.name + shutil.copy2(nb, nb_copy) result = subprocess.run( [ "jupyter", @@ -23,16 +41,15 @@ def run_notebook(nb: Path) -> tuple[bool, float, str]: "--to", "notebook", "--ExecutePreprocessor.timeout=1800", - f"--ExecutePreprocessor.kernel_cwd={tmp}", "--output", nb.name, "--output-dir", tmp, - str(nb), + str(nb_copy), ], capture_output=True, text=True, - cwd=nb.parent, + cwd=tmp, ) elapsed = time.monotonic() - start return result.returncode == 0, elapsed, result.stdout + result.stderr @@ -40,14 +57,18 @@ def run_notebook(nb: Path) -> tuple[bool, float, str]: @pytest.mark.notebooks def test_notebooks() -> None: - notebooks = sorted(NOTEBOOKS_DIR.glob("*.ipynb")) + """ + Function to iteratively test whether all notebooks (with exception to the ones to be excluded, which are defined in `BlACKLIST`) execute + without issues. In the case that a notebook execution fails,the function will print out which notebooks fail and the error message. + """ + notebooks = [nb for nb in NOTEBOOKS_DIR.glob("*.ipynb") if nb.stem not in BLACKLIST] if not notebooks: - print(f"No notebooks found in {NOTEBOOKS_DIR}") + print(f"No valid notebooks found in {NOTEBOOKS_DIR}") sys.exit(1) results: list[tuple[str, bool, float]] = [] for nb in notebooks: - print(f" running {nb.name} ...", end="", flush=True) + print(f" running {nb.stem} ...", end="", flush=True) ok, elapsed, output = run_notebook(nb) status = "PASS" if ok else "FAIL" print(f" {status} ({elapsed:.1f}s)") @@ -55,7 +76,6 @@ def test_notebooks() -> None: print(output) results.append((nb.name, ok, elapsed)) - print() failures = [name for name, ok, _ in results if not ok] print(f"{len(notebooks) - len(failures)}/{len(notebooks)} notebooks passed") if failures: From 021cf784ca725104191212b42912c044f7fead29 Mon Sep 17 00:00:00 2001 From: Nakul Santhosh Date: Wed, 1 Jul 2026 14:06:27 +0200 Subject: [PATCH 4/4] fix: fix formatting --- tests/test_notebook.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_notebook.py b/tests/test_notebook.py index 89957f43..e9eea46b 100755 --- a/tests/test_notebook.py +++ b/tests/test_notebook.py @@ -11,11 +11,7 @@ import pytest NOTEBOOKS_DIR = Path(__file__).parent.parent / "docs/contents/notebooks" -BLACKLIST = [ - "extopt", - "ir_spectrum", - "ml_properties" -] +BLACKLIST = ["extopt", "ir_spectrum", "ml_properties"] def run_notebook(nb: Path) -> tuple[bool, float, str]: