From 47b161fa003d10010f6ce973582e88b0040f47ee Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Tue, 29 Jul 2025 21:32:50 +0100 Subject: [PATCH 01/10] Add smoke tests --- pyproject.toml | 3 + tests/smoke/__init__.py | 1 + tests/smoke/test_examples_smoke.py | 95 ++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 tests/smoke/__init__.py create mode 100644 tests/smoke/test_examples_smoke.py diff --git a/pyproject.toml b/pyproject.toml index c3529e85..a693bc56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,6 +110,9 @@ plugboard = "plugboard.cli:app" asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "session" asyncio_default_test_loop_scope = "session" +markers = [ + "smoke: marks tests as smoke tests (deselect with '-m \"not smoke\"')" +] [tool.coverage.run] source = ["plugboard"] diff --git a/tests/smoke/__init__.py b/tests/smoke/__init__.py new file mode 100644 index 00000000..77b6e930 --- /dev/null +++ b/tests/smoke/__init__.py @@ -0,0 +1 @@ +"""Smoke tests package.""" diff --git a/tests/smoke/test_examples_smoke.py b/tests/smoke/test_examples_smoke.py new file mode 100644 index 00000000..331b0dd9 --- /dev/null +++ b/tests/smoke/test_examples_smoke.py @@ -0,0 +1,95 @@ +"""Smoke tests for examples/tutorials Python files.""" + +from pathlib import Path +import subprocess +import sys +from typing import List, Tuple + +import pytest + + +SMOKE_TEST_TIMEOUT = 60 + + +@pytest.fixture(scope="module") +def tutorial_files() -> List[Tuple[Path, Path]]: + """Find all Python files in examples/tutorials.""" + project_root = Path(__file__).parent.parent.parent + tutorials_dir = project_root / "examples" / "tutorials" + + if not tutorials_dir.exists(): + pytest.skip(f"Tutorials directory not found: {tutorials_dir}") + + tutorial_files = [] + for py_file in tutorials_dir.rglob("*.py"): + # Get the directory containing the Python file + working_dir = py_file.parent + tutorial_files.append((py_file, working_dir)) + + if not tutorial_files: + pytest.skip("No Python files found in examples/tutorials") + + return tutorial_files + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + """Dynamically generate test parameters for each tutorial file.""" + if "file_and_dir" in metafunc.fixturenames: + # Get tutorial files + project_root = Path(__file__).parent.parent.parent + tutorials_dir = project_root / "examples" / "tutorials" + + if not tutorials_dir.exists(): + pytest.skip(f"Tutorials directory not found: {tutorials_dir}") + + tutorial_files = [] + for py_file in tutorials_dir.rglob("*.py"): + working_dir = py_file.parent + tutorial_files.append((py_file, working_dir)) + + if not tutorial_files: + pytest.skip("No Python files found in examples/tutorials") + + # Create test IDs for better test output + test_ids = [str(py_file.relative_to(project_root)) for py_file, _ in tutorial_files] + + metafunc.parametrize("file_and_dir", tutorial_files, ids=test_ids) + + +@pytest.mark.smoke +def test_tutorial_file_runs(file_and_dir: Tuple[Path, Path]) -> None: + """Test that a tutorial file runs without errors.""" + py_file, working_dir = file_and_dir + + try: + # Use subprocess.Popen with cwd argument + process = subprocess.Popen( # noqa: S603 + [sys.executable, py_file.name], + cwd=working_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + try: + stdout, stderr = process.communicate(timeout=SMOKE_TEST_TIMEOUT) + except subprocess.TimeoutExpired: + process.kill() + stdout, stderr = process.communicate() + project_root = Path(__file__).parent.parent.parent + pytest.skip( + f"{py_file.relative_to(project_root)} timed out after {SMOKE_TEST_TIMEOUT} seconds" + ) + + if process.returncode != 0: + project_root = Path(__file__).parent.parent.parent + error_msg = ( + f"Tutorial file {py_file.relative_to(project_root)} " + f"failed to run successfully.\n" + f"Return code: {process.returncode}\n" + f"STDOUT:\n{stdout}\n" + f"STDERR:\n{stderr}" + ) + pytest.fail(error_msg) + except Exception as e: + project_root = Path(__file__).parent.parent.parent + pytest.fail(f"Error running tutorial file {py_file.relative_to(project_root)}: {e}") From e379b3d5b62f3eabae8e63688bd0a5f27e39a848 Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Wed, 30 Jul 2025 07:51:52 +0100 Subject: [PATCH 02/10] Increase timeout --- tests/smoke/test_examples_smoke.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/smoke/test_examples_smoke.py b/tests/smoke/test_examples_smoke.py index 331b0dd9..8b09a013 100644 --- a/tests/smoke/test_examples_smoke.py +++ b/tests/smoke/test_examples_smoke.py @@ -8,7 +8,7 @@ import pytest -SMOKE_TEST_TIMEOUT = 60 +SMOKE_TEST_TIMEOUT = 90 @pytest.fixture(scope="module") From 69133815fbb82ef11813da65b416fda7ab6fec9f Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Wed, 30 Jul 2025 07:55:17 +0100 Subject: [PATCH 03/10] Add smoke tests to CI --- .github/workflows/lint-test.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index f5abd320..c60564f8 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -182,6 +182,34 @@ jobs: include-hidden-files: true path: .coverage.py${{ matrix.python_version }}.integration.tuner* + test-smoke: + name: Tests - smoke + runs-on: ubuntu-latest + timeout-minutes: 5 + strategy: + matrix: + python_version: [3.12, 3.13] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install python + uses: actions/setup-python@v5 + with: + python-version: ${{matrix.python_version}} + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + + - name: Install project + run: uv sync --group test + + - name: Run smoke tests + run: uv run coverage run -m pytest ./tests/smoke/" + coverage-report: name: Report coverage needs: [test-unit, test-integration, test-integration-tuner] # Depends on tests passing From 595f0cb816473c6d7514a55e36b05861ff9d4bda Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Wed, 30 Jul 2025 08:01:08 +0100 Subject: [PATCH 04/10] Typo --- .github/workflows/lint-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index c60564f8..5a2c4f44 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -208,7 +208,7 @@ jobs: run: uv sync --group test - name: Run smoke tests - run: uv run coverage run -m pytest ./tests/smoke/" + run: uv run coverage run -m pytest ./tests/smoke/ coverage-report: name: Report coverage From c743deb2edfc8200579f2c636f4ff14645b7a2bf Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Thu, 7 Aug 2025 20:53:49 +0100 Subject: [PATCH 05/10] Apply changes from code review --- tests/smoke/test_examples_smoke.py | 38 ++++++------------------------ 1 file changed, 7 insertions(+), 31 deletions(-) diff --git a/tests/smoke/test_examples_smoke.py b/tests/smoke/test_examples_smoke.py index 8b09a013..a0247bc1 100644 --- a/tests/smoke/test_examples_smoke.py +++ b/tests/smoke/test_examples_smoke.py @@ -3,41 +3,20 @@ from pathlib import Path import subprocess import sys -from typing import List, Tuple +from typing import Tuple import pytest SMOKE_TEST_TIMEOUT = 90 - - -@pytest.fixture(scope="module") -def tutorial_files() -> List[Tuple[Path, Path]]: - """Find all Python files in examples/tutorials.""" - project_root = Path(__file__).parent.parent.parent - tutorials_dir = project_root / "examples" / "tutorials" - - if not tutorials_dir.exists(): - pytest.skip(f"Tutorials directory not found: {tutorials_dir}") - - tutorial_files = [] - for py_file in tutorials_dir.rglob("*.py"): - # Get the directory containing the Python file - working_dir = py_file.parent - tutorial_files.append((py_file, working_dir)) - - if not tutorial_files: - pytest.skip("No Python files found in examples/tutorials") - - return tutorial_files +PROJECT_ROOT = Path(__file__).parent.parent.parent def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: """Dynamically generate test parameters for each tutorial file.""" if "file_and_dir" in metafunc.fixturenames: # Get tutorial files - project_root = Path(__file__).parent.parent.parent - tutorials_dir = project_root / "examples" / "tutorials" + tutorials_dir = PROJECT_ROOT / "examples" / "tutorials" if not tutorials_dir.exists(): pytest.skip(f"Tutorials directory not found: {tutorials_dir}") @@ -51,7 +30,7 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: pytest.skip("No Python files found in examples/tutorials") # Create test IDs for better test output - test_ids = [str(py_file.relative_to(project_root)) for py_file, _ in tutorial_files] + test_ids = [str(py_file.relative_to(PROJECT_ROOT)) for py_file, _ in tutorial_files] metafunc.parametrize("file_and_dir", tutorial_files, ids=test_ids) @@ -75,15 +54,13 @@ def test_tutorial_file_runs(file_and_dir: Tuple[Path, Path]) -> None: except subprocess.TimeoutExpired: process.kill() stdout, stderr = process.communicate() - project_root = Path(__file__).parent.parent.parent pytest.skip( - f"{py_file.relative_to(project_root)} timed out after {SMOKE_TEST_TIMEOUT} seconds" + f"{py_file.relative_to(PROJECT_ROOT)} timed out after {SMOKE_TEST_TIMEOUT} seconds" ) if process.returncode != 0: - project_root = Path(__file__).parent.parent.parent error_msg = ( - f"Tutorial file {py_file.relative_to(project_root)} " + f"Tutorial file {py_file.relative_to(PROJECT_ROOT)} " f"failed to run successfully.\n" f"Return code: {process.returncode}\n" f"STDOUT:\n{stdout}\n" @@ -91,5 +68,4 @@ def test_tutorial_file_runs(file_and_dir: Tuple[Path, Path]) -> None: ) pytest.fail(error_msg) except Exception as e: - project_root = Path(__file__).parent.parent.parent - pytest.fail(f"Error running tutorial file {py_file.relative_to(project_root)}: {e}") + pytest.fail(f"Error running tutorial file {py_file.relative_to(PROJECT_ROOT)}: {e}") From 9d89055ba0b1692cf9e410163c77fdc18103a56e Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Thu, 7 Aug 2025 20:58:54 +0100 Subject: [PATCH 06/10] Add API key --- .github/workflows/lint-test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index 5a2c4f44..f931a17d 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -209,6 +209,8 @@ jobs: - name: Run smoke tests run: uv run coverage run -m pytest ./tests/smoke/ + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} coverage-report: name: Report coverage From 7c2435f9ef6448004715de65ba25a4a3fcebeeda Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Thu, 7 Aug 2025 21:39:57 +0100 Subject: [PATCH 07/10] Remove comment --- tests/smoke/test_examples_smoke.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/smoke/test_examples_smoke.py b/tests/smoke/test_examples_smoke.py index a0247bc1..de802472 100644 --- a/tests/smoke/test_examples_smoke.py +++ b/tests/smoke/test_examples_smoke.py @@ -41,7 +41,6 @@ def test_tutorial_file_runs(file_and_dir: Tuple[Path, Path]) -> None: py_file, working_dir = file_and_dir try: - # Use subprocess.Popen with cwd argument process = subprocess.Popen( # noqa: S603 [sys.executable, py_file.name], cwd=working_dir, From a10f188469cc0cbda24816cfa8d7e745f4dac787 Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Fri, 8 Aug 2025 19:53:55 +0100 Subject: [PATCH 08/10] Remove coverage on smoke tests --- .github/workflows/lint-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index f931a17d..2c40944f 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -208,7 +208,7 @@ jobs: run: uv sync --group test - name: Run smoke tests - run: uv run coverage run -m pytest ./tests/smoke/ + run: uv run pytest ./tests/smoke/ --working_dir . env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} From 91c41b23e306b031099f16f6e3e45566d405d1b1 Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Fri, 8 Aug 2025 19:54:07 +0100 Subject: [PATCH 09/10] Fix issue with ray uv integration --- tests/smoke/test_examples_smoke.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/smoke/test_examples_smoke.py b/tests/smoke/test_examples_smoke.py index de802472..7d1db7ce 100644 --- a/tests/smoke/test_examples_smoke.py +++ b/tests/smoke/test_examples_smoke.py @@ -1,9 +1,10 @@ """Smoke tests for examples/tutorials Python files.""" +import os from pathlib import Path import subprocess import sys -from typing import Tuple +from typing import Iterator, Tuple import pytest @@ -12,6 +13,16 @@ PROJECT_ROOT = Path(__file__).parent.parent.parent +@pytest.fixture(scope="module", autouse=True) +def ray_disable_uv_run() -> Iterator[None]: + """Disable Ray's `uv run` runtime environment for smoke tests.""" + # uv run environment will prevent tests from running outside of the project root + # This is necessary because the smoke tests run in a separate process + os.environ["RAY_ENABLE_UV_RUN_RUNTIME_ENV"] = "0" + yield + os.environ.pop("RAY_ENABLE_UV_RUN_RUNTIME_ENV", None) + + def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: """Dynamically generate test parameters for each tutorial file.""" if "file_and_dir" in metafunc.fixturenames: From 2a5510158f895aa4335a63f10555ae0abb70944d Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Fri, 8 Aug 2025 20:02:18 +0100 Subject: [PATCH 10/10] Fixup --- .github/workflows/lint-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index 2c40944f..b40b8eb4 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -208,7 +208,7 @@ jobs: run: uv sync --group test - name: Run smoke tests - run: uv run pytest ./tests/smoke/ --working_dir . + run: uv run pytest ./tests/smoke/ env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}