diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index f5abd320..b40b8eb4 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -182,6 +182,36 @@ jobs: include-hidden-files: true path: .coverage.py${{ matrix.python_version }}.integration.tuner* + test-smoke: + name: Tests - smoke + runs-on: ubuntu-latest + timeout-minutes: 5 + strategy: + matrix: + python_version: [3.12, 3.13] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install python + uses: actions/setup-python@v5 + with: + python-version: ${{matrix.python_version}} + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + + - name: Install project + run: uv sync --group test + + - name: Run smoke tests + run: uv run pytest ./tests/smoke/ + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + coverage-report: name: Report coverage needs: [test-unit, test-integration, test-integration-tuner] # Depends on tests passing diff --git a/pyproject.toml b/pyproject.toml index c3529e85..a693bc56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,6 +110,9 @@ plugboard = "plugboard.cli:app" asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "session" asyncio_default_test_loop_scope = "session" +markers = [ + "smoke: marks tests as smoke tests (deselect with '-m \"not smoke\"')" +] [tool.coverage.run] source = ["plugboard"] diff --git a/tests/smoke/__init__.py b/tests/smoke/__init__.py new file mode 100644 index 00000000..77b6e930 --- /dev/null +++ b/tests/smoke/__init__.py @@ -0,0 +1 @@ +"""Smoke tests package.""" diff --git a/tests/smoke/test_examples_smoke.py b/tests/smoke/test_examples_smoke.py new file mode 100644 index 00000000..7d1db7ce --- /dev/null +++ b/tests/smoke/test_examples_smoke.py @@ -0,0 +1,81 @@ +"""Smoke tests for examples/tutorials Python files.""" + +import os +from pathlib import Path +import subprocess +import sys +from typing import Iterator, Tuple + +import pytest + + +SMOKE_TEST_TIMEOUT = 90 +PROJECT_ROOT = Path(__file__).parent.parent.parent + + +@pytest.fixture(scope="module", autouse=True) +def ray_disable_uv_run() -> Iterator[None]: + """Disable Ray's `uv run` runtime environment for smoke tests.""" + # uv run environment will prevent tests from running outside of the project root + # This is necessary because the smoke tests run in a separate process + os.environ["RAY_ENABLE_UV_RUN_RUNTIME_ENV"] = "0" + yield + os.environ.pop("RAY_ENABLE_UV_RUN_RUNTIME_ENV", None) + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + """Dynamically generate test parameters for each tutorial file.""" + if "file_and_dir" in metafunc.fixturenames: + # Get tutorial files + tutorials_dir = PROJECT_ROOT / "examples" / "tutorials" + + if not tutorials_dir.exists(): + pytest.skip(f"Tutorials directory not found: {tutorials_dir}") + + tutorial_files = [] + for py_file in tutorials_dir.rglob("*.py"): + working_dir = py_file.parent + tutorial_files.append((py_file, working_dir)) + + if not tutorial_files: + pytest.skip("No Python files found in examples/tutorials") + + # Create test IDs for better test output + test_ids = [str(py_file.relative_to(PROJECT_ROOT)) for py_file, _ in tutorial_files] + + metafunc.parametrize("file_and_dir", tutorial_files, ids=test_ids) + + +@pytest.mark.smoke +def test_tutorial_file_runs(file_and_dir: Tuple[Path, Path]) -> None: + """Test that a tutorial file runs without errors.""" + py_file, working_dir = file_and_dir + + try: + process = subprocess.Popen( # noqa: S603 + [sys.executable, py_file.name], + cwd=working_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + try: + stdout, stderr = process.communicate(timeout=SMOKE_TEST_TIMEOUT) + except subprocess.TimeoutExpired: + process.kill() + stdout, stderr = process.communicate() + pytest.skip( + f"{py_file.relative_to(PROJECT_ROOT)} timed out after {SMOKE_TEST_TIMEOUT} seconds" + ) + + if process.returncode != 0: + error_msg = ( + f"Tutorial file {py_file.relative_to(PROJECT_ROOT)} " + f"failed to run successfully.\n" + f"Return code: {process.returncode}\n" + f"STDOUT:\n{stdout}\n" + f"STDERR:\n{stderr}" + ) + pytest.fail(error_msg) + except Exception as e: + pytest.fail(f"Error running tutorial file {py_file.relative_to(PROJECT_ROOT)}: {e}")