Merge pull request #331 from Ayaan-20-11/fix/cross-platform-terminal-… #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Audits | |
| # Smoke-test every audit so a broken dataset path or script | |
| # (like the AI Fair Recruitment FileNotFoundError) is caught automatically. | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| jobs: | |
| changes: | |
| # Decides whether `profiler`/`benchmark-harness` below need to run at | |
| # all - a docs- or website-only push gets no value from re-running the | |
| # test suite. `run-audits` is deliberately NOT gated by this: it's the | |
| # one job in this workflow that's a required status check (see #160), | |
| # and a job skipped via `if:` can leave a required check stuck | |
| # "Expected - waiting for status to be reported" instead of green, | |
| # which would block merging every docs-only PR. Skipping a | |
| # non-required job has no such risk - it just shows as neutral/skipped. | |
| # | |
| # Path set mirrors the pre-push pytest hook in .pre-commit-config.yaml | |
| # (#237) - same question ("could this change affect the test suite?"), | |
| # same answer. Keep the two in sync. | |
| runs-on: ubuntu-latest | |
| outputs: | |
| code_relevant: ${{ steps.filter.outputs.code_relevant }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 0 | |
| - name: Detect whether this change could affect the profiler/benchmark tests | |
| id: filter | |
| run: | | |
| set -euo pipefail | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| base="${{ github.event.pull_request.base.sha }}" | |
| else | |
| base="${{ github.event.before }}" | |
| fi | |
| if [ -z "$base" ] || [ "$base" = "0000000000000000000000000000000000000000" ] || ! git cat-file -e "$base" 2>/dev/null; then | |
| echo "No usable base commit (new branch, force-push, or first push) - treating as relevant." | |
| echo "code_relevant=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| changed="$(git diff --name-only "$base" "${{ github.sha }}")" | |
| echo "Changed files:" | |
| echo "$changed" | |
| if echo "$changed" | grep -Eq '^(faircode/.*|scripts/.*|tests/.*|assets/(profiler-engine\.js|profiler-compare\.js)|pyproject\.toml|requirements(-lock)?\.txt|\.github/CODEOWNERS|.*\.csv|.*/(fair|unfair)\.py|.*/audit\.yaml)$'; then | |
| echo "code_relevant=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "code_relevant=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| run-audits: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| audit: | |
| - "COMPAS" | |
| - "AI Fair Recruitment" | |
| - "German Credit Lending" | |
| - "Insurance Denial" | |
| - "Benefits Denial" | |
| - "Healthcare Readmission" | |
| - "Tenant Screening" | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: actions/setup-python@v7 | |
| with: | |
| python-version: '3.12' | |
| cache: 'pip' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| pip install -e . | |
| - name: Run ${{ matrix.audit }} audit scripts | |
| run: | | |
| set -e | |
| echo "::group::${{ matrix.audit }} - unfair.py" | |
| python "${{ matrix.audit }}/unfair.py" | |
| echo "::endgroup::" | |
| echo "::group::${{ matrix.audit }} - fair.py" | |
| python "${{ matrix.audit }}/fair.py" | |
| echo "::endgroup::" | |
| profiler: | |
| # Test + smoke-test the faircode dataset profiler (the diagnostic tool). | |
| # Not a required status check - safe to skip when nothing test-relevant | |
| # changed. See the `changes` job above. | |
| needs: changes | |
| if: needs.changes.outputs.code_relevant == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: actions/setup-python@v7 | |
| with: | |
| python-version: '3.12' | |
| cache: 'pip' | |
| - uses: actions/setup-node@v7 | |
| with: | |
| node-version: '20' | |
| - name: Install faircode + pytest | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[excel,parquet,proxy,mcp]" pytest pytest-cov | |
| - name: Run profiler tests | |
| run: make test | |
| - name: Report test coverage | |
| run: make coverage | |
| continue-on-error: true | |
| - name: Smoke-test the CLI | |
| run: | | |
| set -e | |
| faircode profile "Insurance Denial/insurance.csv" | |
| faircode profile "Benefits Denial/adult.csv" --json > /dev/null | |
| benchmark-harness: | |
| # Runs the benchmark harness's own tests plus a real end-to-end pass over | |
| # ONE small audit, so a contributed audit.yaml (or a change to | |
| # manifest.py/strategies.py/models.py/metrics.py/benchmark.py) that | |
| # breaks the pipeline is caught in the PR. The full seven-domain sweep is | |
| # deliberately NOT run here - fairlearn's in-processing strategy refits | |
| # its base estimator multiple times per (audit, model) cell and takes | |
| # minutes on the larger datasets. See "Reproducibility & Paper Freeze" in | |
| # README.md: run the full benchmark locally and commit results/ output. | |
| # | |
| # Not a required status check - safe to skip when nothing test-relevant | |
| # changed. See the `changes` job above. | |
| needs: changes | |
| if: needs.changes.outputs.code_relevant == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: actions/setup-python@v7 | |
| with: | |
| python-version: '3.12' | |
| cache: 'pip' | |
| - name: Install faircode[benchmark] + pytest | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[benchmark]" pytest | |
| - name: Run benchmark-harness unit + integration tests | |
| run: pytest tests/test_metrics.py tests/test_manifest.py tests/test_strategies.py tests/test_benchmark.py -v | |
| - name: Smoke-test the CLI end-to-end on one small audit | |
| run: | | |
| set -e | |
| faircode benchmark "German Credit Lending/audit.yaml" \ | |
| --n-resamples 100 --n-permutations 100 --no-plots \ | |
| --out /tmp/benchmark-smoke | |
| test -s /tmp/benchmark-smoke/results_fairness.csv | |
| test -s /tmp/benchmark-smoke/results_performance.csv | |
| echo "Benchmark harness smoke test passed: $(wc -l < /tmp/benchmark-smoke/results_fairness.csv) fairness rows written" |