diff --git a/.github/workflows/ci-audit.yml b/.github/workflows/ci-audit.yml new file mode 100644 index 0000000..b4fe40d --- /dev/null +++ b/.github/workflows/ci-audit.yml @@ -0,0 +1,113 @@ +name: CI Audit - Tests and Code Quality + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main, develop ] + +# Set minimal permissions for GITHUB_TOKEN +permissions: + contents: read + +jobs: + test: + name: Run Tests + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11"] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive # Initialize any git submodules if present + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + + # Install test dependencies + pip install pytest pytest-cov pytest-xdist + + # Install GA_denoise_selector dependencies + if [ -f GA_denoise_selector/requirements.txt ]; then + pip install -r GA_denoise_selector/requirements.txt + fi + + # Install Mamba-SSM-Denoise dependencies + if [ -f Mamba-SSM-Denoise/Track_H_cross_validation/requirements.txt ]; then + pip install -r Mamba-SSM-Denoise/Track_H_cross_validation/requirements.txt + fi + + # Install GA_smoother_classic dependencies + if [ -f GA_smoother_classic/requirements.txt ]; then + pip install -r GA_smoother_classic/requirements.txt + fi + + - name: Set PYTHONPATH + run: | + echo "PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd)/GA_denoise_selector:$(pwd)/Mamba-SSM-Denoise/Track_H_cross_validation:$(pwd)/GA_smoother_classic" >> $GITHUB_ENV + + - name: Run tests + run: | + # Export PYTHONPATH for pytest (belt and suspenders approach) + export PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd)/GA_denoise_selector:$(pwd)/Mamba-SSM-Denoise/Track_H_cross_validation:$(pwd)/GA_smoother_classic + + # Run tests with quick marker if available, otherwise run all tests + if grep -r "pytest.mark.quick" tests/ 2>/dev/null; then + pytest tests -m "quick" --maxfail=1 --cov=GA_denoise_selector/smoothers --cov=Mamba-SSM-Denoise/Track_H_cross_validation/src --cov-report=term --cov-report=html + else + pytest tests --maxfail=1 --cov=GA_denoise_selector/smoothers --cov=Mamba-SSM-Denoise/Track_H_cross_validation/src --cov-report=term --cov-report=html + fi + + - name: Upload coverage reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: coverage-report-${{ matrix.python-version }} + path: htmlcov/ + retention-days: 30 + + lint: + name: Code Quality Checks + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install linting tools + run: | + python -m pip install --upgrade pip + pip install flake8 black isort mypy + + - name: Run flake8 + continue-on-error: true + run: | + # Stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # Exit-zero treats all errors as warnings + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Check code formatting with black + continue-on-error: true + run: | + black --check --diff . + + - name: Check import ordering with isort + continue-on-error: true + run: | + isort --check-only --diff . diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1357c40 --- /dev/null +++ b/.gitignore @@ -0,0 +1,93 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyTorch +*.pth +*.pt +checkpoints/ +runs/ +lightning_logs/ + +# Testing +.pytest_cache/ +.coverage +.coverage.* +htmlcov/ +.tox/ +.nox/ +coverage.xml +*.cover +.hypothesis/ +pytest_cache/ + +# Jupyter Notebook +.ipynb_checkpoints +*.ipynb + +# Environment +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# IDEs +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store + +# Data +*.csv +*.h5 +*.hdf5 +*.pkl +*.pickle +data/ +datasets/ +output/ +outputs/ +results/ + +# Logs +*.log +logs/ +tensorboard/ + +# Git submodules (if any future submodules are added) +external/ + +# Temporary files +tmp/ +temp/ +*.tmp + +# OS-specific +.DS_Store +Thumbs.db diff --git a/README.md b/README.md index 358174f..e7113e9 100644 --- a/README.md +++ b/README.md @@ -3,3 +3,58 @@ This repository collects quick experiments and reusable components for transformer architectures and state-space model (SSM) projects. Expect lightweight prototypes, training scripts, and utility code that make it easy to compare and iterate on modern sequence modeling ideas. Jump in, explore the models, and use the pieces that help you move faster on your own AI workflows. + +## Projects + +- **GA_denoise_selector**: Genetic algorithm-based smoother selection with 13+ filter implementations +- **Mamba-SSM-Denoise**: Deep learning denoising using Mamba state-space models +- **GA_smoother_classic**: Classic genetic algorithm smoother implementation + +## Testing + +This repository includes a test infrastructure to ensure code quality and prevent import errors. + +### Running Tests + +```bash +# Install test dependencies +pip install pytest pytest-cov + +# Run all tests +pytest tests/ + +# Run quick tests only +pytest tests/ -m "quick" + +# Run with coverage +pytest tests/ --cov=GA_denoise_selector/smoothers --cov=Mamba-SSM-Denoise/Track_H_cross_validation/src +``` + +### Configuration + +- **pytest configuration**: `setup.cfg` - Configures Python path for all projects +- **Test directory**: `tests/` - All test files +- **CI/CD**: `.github/workflows/ci-audit.yml` - Automated testing on push/PR + +For more details, see [tests/README.md](tests/README.md). + +## Development + +### Prerequisites + +Each project has its own dependencies. Install them as needed: + +```bash +# GA_denoise_selector +pip install -r GA_denoise_selector/requirements.txt + +# Mamba-SSM-Denoise +pip install -r Mamba-SSM-Denoise/Track_H_cross_validation/requirements.txt + +# GA_smoother_classic +pip install -r GA_smoother_classic/requirements.txt +``` + +### Python Path + +The repository uses a multi-project structure. When importing modules, the Python path is configured via `setup.cfg` to include all project directories. This prevents `ModuleNotFoundError` issues in tests. diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..7fe9ac2 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,38 @@ +[tool:pytest] +# Configure pytest to find source modules for all projects +pythonpath = + . + GA_denoise_selector + Mamba-SSM-Denoise/Track_H_cross_validation + GA_smoother_classic + +# Test discovery patterns +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* + +# Custom markers +markers = + quick: marks tests as quick (deselect with '-m "not quick"') + +# Coverage configuration +[coverage:run] +source = + GA_denoise_selector/smoothers + Mamba-SSM-Denoise/Track_H_cross_validation/src + +omit = + */tests/* + */test_*.py + */__pycache__/* + +[coverage:report] +exclude_lines = + pragma: no cover + def __repr__ + raise AssertionError + raise NotImplementedError + if __name__ == '__main__': + if TYPE_CHECKING: + @abstractmethod diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..66a03c1 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,120 @@ +# Tests Directory + +This directory contains test cases for the AI repository projects. + +## Structure + +``` +tests/ +├── __init__.py +├── reflectance/ +│ ├── __init__.py +│ └── test_dose_metadata.py # Tests for dose metadata handling +└── README.md # This file +``` + +## Running Tests + +### Prerequisites + +1. Install pytest and coverage tools: +```bash +pip install pytest pytest-cov pytest-xdist +``` + +2. Install project dependencies: +```bash +# GA_denoise_selector dependencies +pip install -r GA_denoise_selector/requirements.txt + +# Mamba-SSM-Denoise dependencies +pip install -r Mamba-SSM-Denoise/Track_H_cross_validation/requirements.txt + +# GA_smoother_classic dependencies +pip install -r GA_smoother_classic/requirements.txt +``` + +### Running All Tests + +From the repository root: + +```bash +pytest tests/ +``` + +### Running Quick Tests Only + +```bash +pytest tests/ -m "quick" +``` + +### Running with Coverage + +```bash +pytest tests/ --cov=GA_denoise_selector/smoothers --cov=Mamba-SSM-Denoise/Track_H_cross_validation/src --cov-report=term --cov-report=html +``` + +## Configuration + +The test configuration is defined in `setup.cfg` at the repository root: + +- **Python path**: Configured to include all project directories +- **Test discovery**: Automatically finds `test_*.py` files +- **Coverage**: Tracks coverage for main source packages + +## Writing Tests + +### Import Validation Tests + +Tests that verify the Python path is correctly configured: + +```python +def test_import_src_package(): + """Test that src package can be imported.""" + from src import datasets + assert datasets is not None +``` + +### Quick Tests + +Mark tests that should run quickly for rapid feedback: + +```python +@pytest.mark.quick +def test_basic_validation(): + """Quick validation test.""" + assert True +``` + +## CI/CD Integration + +Tests are automatically run in GitHub Actions via `.github/workflows/ci-audit.yml`: + +- Runs on Python 3.10 and 3.11 +- Executes quick tests with the `-m "quick"` marker +- Generates coverage reports +- Uploads coverage artifacts + +## Troubleshooting + +### ModuleNotFoundError + +If you encounter `ModuleNotFoundError: No module named 'src'` or similar: + +1. Verify `setup.cfg` includes the correct pythonpath entries +2. Check that `PYTHONPATH` environment variable is set correctly: +```bash +export PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd)/GA_denoise_selector:$(pwd)/Mamba-SSM-Denoise/Track_H_cross_validation +``` + +3. Ensure you're running pytest from the repository root directory + +### Import Path Issues + +The repository has multiple Python projects with different structures: + +- **GA_denoise_selector**: Import as `from smoothers import ...` +- **Mamba-SSM-Denoise**: Import as `from src import ...` or `from src.models import ...` +- **GA_smoother_classic**: Import as `import ga_smoother` + +Make sure your imports match the package structure. diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..0f6bc8c --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the AI repository.""" diff --git a/tests/reflectance/__init__.py b/tests/reflectance/__init__.py new file mode 100644 index 0000000..1cbfe39 --- /dev/null +++ b/tests/reflectance/__init__.py @@ -0,0 +1 @@ +"""Tests for reflectance-related functionality.""" diff --git a/tests/reflectance/test_dose_metadata.py b/tests/reflectance/test_dose_metadata.py new file mode 100644 index 0000000..c7c4629 --- /dev/null +++ b/tests/reflectance/test_dose_metadata.py @@ -0,0 +1,56 @@ +""" +Tests for dose metadata handling. + +This test file validates that the Python path is correctly configured +and that modules from the src package can be imported properly. +""" + +import pytest + + +# Test that we can import from src package (Mamba-SSM-Denoise) +def test_import_src_package(): + """Test that src package can be imported without ModuleNotFoundError.""" + try: + from src import datasets + assert datasets is not None + except ModuleNotFoundError as e: + pytest.fail(f"Failed to import src package: {e}") + + +# Test that we can import from smoothers package (GA_denoise_selector) +def test_import_smoothers_package(): + """Test that smoothers package can be imported.""" + try: + from smoothers import SGParams, apply_sg + assert SGParams is not None + assert apply_sg is not None + except ModuleNotFoundError as e: + pytest.fail(f"Failed to import smoothers package: {e}") + + +@pytest.mark.quick +def test_basic_import_validation(): + """Quick test to verify basic import functionality.""" + # Test importing standard library + import sys + import os + + # Verify Python path includes expected directories + pythonpath_str = os.environ.get('PYTHONPATH', '') + print(f"PYTHONPATH: {pythonpath_str}") + print(f"sys.path: {sys.path}") + + assert sys is not None + assert os is not None + + +@pytest.mark.quick +def test_dose_metadata_placeholder(): + """Placeholder test for dose metadata functionality. + + This is a minimal test to verify the test infrastructure is working. + Replace with actual dose metadata tests as needed. + """ + # This is a placeholder - replace with actual dose metadata logic + assert True