audits.yml

name: Audits

# Smoke-test every audit so a broken dataset path or script
# (like the AI Fair Recruitment FileNotFoundError) is caught automatically.

on:
  push:
    branches: [ main ]
  pull_request:

jobs:
  changes:
    # Decides whether `profiler`/`benchmark-harness` below need to run at
    # all - a docs- or website-only push gets no value from re-running the
    # test suite. `run-audits` is deliberately NOT gated by this: it's the
    # one job in this workflow that's a required status check (see #160),
    # and a job skipped via `if:` can leave a required check stuck
    # "Expected - waiting for status to be reported" instead of green,
    # which would block merging every docs-only PR. Skipping a
    # non-required job has no such risk - it just shows as neutral/skipped.
    #
    # Path set mirrors the pre-push pytest hook in .pre-commit-config.yaml
    # (#237) - same question ("could this change affect the test suite?"),
    # same answer. Keep the two in sync.
    runs-on: ubuntu-latest
    outputs:
      code_relevant: ${{ steps.filter.outputs.code_relevant }}
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 0

      - name: Detect whether this change could affect the profiler/benchmark tests
        id: filter
        run: |
          set -euo pipefail

          if [ "${{ github.event_name }}" = "pull_request" ]; then
            base="${{ github.event.pull_request.base.sha }}"
          else
            base="${{ github.event.before }}"
          fi

          if [ -z "$base" ] || [ "$base" = "0000000000000000000000000000000000000000" ] || ! git cat-file -e "$base" 2>/dev/null; then
            echo "No usable base commit (new branch, force-push, or first push) - treating as relevant."
            echo "code_relevant=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          changed="$(git diff --name-only "$base" "${{ github.sha }}")"
          echo "Changed files:"
          echo "$changed"

          if echo "$changed" | grep -Eq '^(faircode/.*|scripts/.*|tests/.*|assets/(profiler-engine\.js|profiler-compare\.js)|pyproject\.toml|requirements(-lock)?\.txt|\.github/CODEOWNERS|.*\.csv|.*/(fair|unfair)\.py|.*/audit\.yaml)$'; then
            echo "code_relevant=true" >> "$GITHUB_OUTPUT"
          else
            echo "code_relevant=false" >> "$GITHUB_OUTPUT"
          fi

  run-audits:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        audit:
          - "COMPAS"
          - "AI Fair Recruitment"
          - "German Credit Lending"
          - "Insurance Denial"
          - "Benefits Denial"
          - "Healthcare Readmission"
          - "Tenant Screening"
    steps:
      - uses: actions/checkout@v7

      - uses: actions/setup-python@v7
        with:
          python-version: '3.12'
          cache: 'pip'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt
          pip install -e .

      - name: Run ${{ matrix.audit }} audit scripts
        run: |
          set -e
          echo "::group::${{ matrix.audit }} - unfair.py"
          python "${{ matrix.audit }}/unfair.py"
          echo "::endgroup::"
          echo "::group::${{ matrix.audit }} - fair.py"
          python "${{ matrix.audit }}/fair.py"
          echo "::endgroup::"

  profiler:
    # Test + smoke-test the faircode dataset profiler (the diagnostic tool).
    # Not a required status check - safe to skip when nothing test-relevant
    # changed. See the `changes` job above.
    needs: changes
    if: needs.changes.outputs.code_relevant == 'true'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7

      - uses: actions/setup-python@v7
        with:
          python-version: '3.12'
          cache: 'pip'

      - uses: actions/setup-node@v7
        with:
          node-version: '20'

      - name: Install faircode + pytest
        run: |
          python -m pip install --upgrade pip
          pip install -e ".[excel,parquet,proxy,mcp]" pytest pytest-cov

      - name: Run profiler tests
        run: make test

      - name: Report test coverage
        run: make coverage
        continue-on-error: true

      - name: Smoke-test the CLI
        run: |
          set -e
          faircode profile "Insurance Denial/insurance.csv"
          faircode profile "Benefits Denial/adult.csv" --json > /dev/null

  benchmark-harness:
    # Runs the benchmark harness's own tests plus a real end-to-end pass over
    # ONE small audit, so a contributed audit.yaml (or a change to
    # manifest.py/strategies.py/models.py/metrics.py/benchmark.py) that
    # breaks the pipeline is caught in the PR. The full seven-domain sweep is
    # deliberately NOT run here - fairlearn's in-processing strategy refits
    # its base estimator multiple times per (audit, model) cell and takes
    # minutes on the larger datasets. See "Reproducibility & Paper Freeze" in
    # README.md: run the full benchmark locally and commit results/ output.
    #
    # Not a required status check - safe to skip when nothing test-relevant
    # changed. See the `changes` job above.
    needs: changes
    if: needs.changes.outputs.code_relevant == 'true'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7

      - uses: actions/setup-python@v7
        with:
          python-version: '3.12'
          cache: 'pip'

      - name: Install faircode[benchmark] + pytest
        run: |
          python -m pip install --upgrade pip
          pip install -e ".[benchmark]" pytest

      - name: Run benchmark-harness unit + integration tests
        run: pytest tests/test_metrics.py tests/test_manifest.py tests/test_strategies.py tests/test_benchmark.py -v

      - name: Smoke-test the CLI end-to-end on one small audit
        run: |
          set -e
          faircode benchmark "German Credit Lending/audit.yaml" \
            --n-resamples 100 --n-permutations 100 --no-plots \
            --out /tmp/benchmark-smoke
          test -s /tmp/benchmark-smoke/results_fairness.csv
          test -s /tmp/benchmark-smoke/results_performance.csv
          echo "Benchmark harness smoke test passed: $(wc -l < /tmp/benchmark-smoke/results_fairness.csv) fairness rows written"