ci.yml

# CI: tests and dependency audit on push/PR.
# Keeps main branch healthy and improves security via automated checks.
#
# Failure posture (fail-closed by default):
#   - Individual steps: every step uses the default `continue-on-error: false`.
#     A failed step (non-zero exit code) immediately stops that job and marks
#     the run as failed — the pipeline does NOT silently pass on errors.
#   - `uv sync` / `uv pip install pip-audit` network failures: if dependency
#     fetch or lock-file install fails (network timeout, registry unavailable,
#     hash mismatch), the job exits with a non-zero code and the entire run
#     is marked FAILED (not skipped, not passed).
#   - `pip-audit` security scan: runs after install; a discovered vulnerability
#     without a matching `--ignore-vuln` entry fails the job.
#   - `fail-fast: false` on the matrix means Python-3.12/3.13/3.14 jobs
#     run to completion even if one fails, so all error logs are collected.
#     It does NOT mean "pass when there are errors" — just parallel visibility.
#   - Python 3.14 is SIGNAL-ONLY in the CI matrix (#551): it runs the full test
#     suite for the no-GIL Enterprise runtime but uses `continue-on-error: true`
#     so an ecosystem gap (missing wheels, dep incompatibility) does not block
#     merges. The Docker release base image is now python:3.14-slim (licensing
#     enforcement gate green — #551 worker cap + closed cluster #704/#717–#722/#856;
#     field matrix Alpine/Void cp314). Matrix continue-on-error for 3.14 is a
#     separate CI posture decision from the image base pin.
#
# Third-party Actions use full commit SHAs (major tag noted in comment). Refresh via Dependabot
# (github-actions) or deliberately after reading upstream release notes. astral-sh/setup-uv `version`
# pins the uv CLI release (see https://github.com/astral-sh/uv/releases), not the action repo tag.
#
# Explicit permissions limit GITHUB_TOKEN (satisfies "Workflow does not contain permissions" check).
# contents: read is the minimal scope needed for checkout and dependency install.
name: CI

permissions:
  contents: read

on:
  push:
    branches: [main, master]
  pull_request:
    branches: [main, master]

jobs:
  test:
    name: Test (Python ${{ matrix.python-version }})
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.12", "3.13", "3.14"]
    # 3.14 is signal-only (#551) — see header comment. 3.12/3.13 stay blocking.
    continue-on-error: ${{ matrix.python-version == '3.14' }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          # Full history required for scripts/pii_history_guard.py --full-history (PII in old commits).
          fetch-depth: 0
          persist-credentials: false

      - name: Install libmariadb-dev (mariadb PyPI connector builds from source on Linux)
        uses: $/.github/actions/install-libmariadb-dev

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: ${{ matrix.python-version }}

      - name: Try install uv via action (non-fatal)
        id: setup_uv
        continue-on-error: true
        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
        with:
          version: "0.11.2"

      - name: Fallback install uv via pip if action failed
        if: steps.setup_uv.outcome != 'success'
        run: |
          python -m pip install --disable-pip-version-check --require-hashes \
            -r .github/pip-constraints/ci-uv-fallback.txt
          uv --version

      - name: Install dependencies
        # shares extra: smbprotocol/webdav — keeps SMB/CIFS/WebDAV connector registration aligned with TECH_GUIDE
        # dev group: pytest, hypothesis, pre-commit stack — not shipped to end-user `uv sync` default in CI test matrix
        run: uv sync --extra shares --group dev

      - name: Install gitleaks (pinned binary SHA256; kombi PLAN_NO_COAUTHORSHIP_GATE)
        run: |
          set -euo pipefail
          source scripts/tool-pins.sh
          BASE="https://github.com/gitleaks/gitleaks/releases/download/v${DB_GITLEAKS_VERSION}"
          TARBALL="gitleaks_${DB_GITLEAKS_VERSION}_linux_x64.tar.gz"
          MAX_RETRIES=5
          for i in $(seq 1 $MAX_RETRIES); do
            if curl --fail --show-error --location --retry 3 --retry-delay 5 --retry-connrefused \
              "${BASE}/${TARBALL}" -o "${TARBALL}"; then
              break
            fi
            echo "Download attempt $i failed; sleeping $((i * 5))s before retry..."
            sleep $((i * 5))
            if [ "$i" -eq "$MAX_RETRIES" ]; then
              echo "Failed to download ${TARBALL} after ${MAX_RETRIES} attempts" >&2
              exit 1
            fi
          done
          tar -xzf "${TARBALL}" gitleaks
          echo "${DB_GITLEAKS_LINUX_X64_BINARY_SHA256}  gitleaks" | sha256sum -c -
          sudo install -m 0755 gitleaks /usr/local/bin/gitleaks

      - name: Run tests
        if: matrix.python-version != '3.13'
        run: uv run pytest -v -W error
        # addopts in pyproject.toml also set -W error; explicit in CI so tests pass without warnings.

      - name: Run tests with coverage (Python 3.13 → Sonar, #1719)
        if: matrix.python-version == '3.13'
        run: uv run pytest -v -W error --cov --cov-report=xml

      - name: Upload coverage.xml for Sonar
        if: matrix.python-version == '3.13'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: coverage-xml
          path: coverage.xml
          if-no-files-found: error

      - name: PII history guard (full git history)
        run: uv run python scripts/pii_history_guard.py --full-history

      - name: PII gate change tripwire (self-protecting gate; ADR-0071, #944)
        run: |
          git fetch origin main --depth=1 2>/dev/null || true
          uv run python scripts/gate_change_tripwire.py --base origin/main

  # Optional extras coverage (#1638): default matrix only installs ``shares``.
  # One extra Ubuntu job installs SQL extras except ``mariadb``, plus
  # nosql/compressed/dataformats so importorskip tests actually run.
  # mariadb 1.1.14 (latest stable on PyPI) has a non-raw docstring with ``\*``
  # in connectionpool.py; Python 3.13 raises SyntaxError (was SyntaxWarning).
  # 2.0.0 is still rc-only — do not pin an RC. Restore ``--extra mariadb`` (or
  # ``sql-all``) when a stable connector imports on 3.13. Also PYSEC-2026-217
  # on 1.1.14 (#922). Skip ceiling measures optional-extra gaps only (Maestro
  # clone-gated tests are deselected; see the pytest step).
  # Extra ``dl`` (torch + sentence-transformers) is a dedicated ``test-dl``
  # job (#1822) so this skip ceiling stays about SQL/nosql extras, not model download.
  test-extras:
    name: Test optional extras (Python 3.13)
    runs-on: ubuntu-latest
    timeout-minutes: 50
    env:
      DATA_BOAR_CI_EXTRAS: "1"
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0
          persist-credentials: false

      - name: Install unixodbc-dev (pyodbc / mssql-pyodbc extra)
        run: timeout 240 sudo apt-get install -y unixodbc-dev

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.13"

      - name: Install uv
        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
        with:
          version: "0.11.2"

      - name: Install dependencies (optional extras coverage)
        # sql-all minus mariadb (3.13 SyntaxError in upstream 1.1.14).
        # shares: same as default test job so SMB/WebDAV guards stay satisfied.
        run: uv sync --extra postgres --extra mysql --extra mssql --extra mssql-pyodbc --extra oracle --extra nosql --extra compressed --extra dataformats --extra shares --group dev

      - name: Run tests
        # Public CI must not clone private DataBoar/maestro: forks have no org
        # PAT, a long-lived token in public CI is the wrong coupling, and
        # spinout maestro#8 skips those consumer guards when the clone is
        # absent ("typical public CI"). Deselect so the skip ceiling measures
        # optional Python extras only. If Maestro guards should run in CI, add
        # a separate opt-in job — do not hang it off test-extras.
        run: |
          uv run pytest -v -W error -ra --tb=short --junitxml=extras-junit.xml \
            --ignore=tests/test_maestro_scripts.py \
            --ignore=tests/test_dl_backend_ci.py \
            --deselect=tests/test_issue_dev_license_qa.py::test_maestro_handler_issues_60d_machine_bound \
            --deselect=tests/test_security.py::test_sync_working_tree_excludes_dotenv_from_rsync \
            --deselect=tests/test_security.py::test_maestro_aggregates_real_failures_in_exit

      - name: Skip-count ceiling (#1638)
        # extras job on 989a4a3f: skipped=106. 56 are MAESTRO_ROOT guards
        # (test_maestro_scripts.py 53 + test_issue_dev_license_qa.py 1 +
        # test_security.py 2). Remainder 50 (JUnit counts 1 xfail as skip).
        # Ceiling 60 = remainder +10 slack for suite growth; not Maestro coverage.
        run: uv run python scripts/ci_pytest_skip_ceiling.py extras-junit.xml --max-skipped 60

  # Optional DL extra (#1822): torch is too heavy for test-extras. This job
  # installs ``--extra dl`` and runs a minimal encode through core/dl_backend.py.
  test-dl:
    name: Test DL extra (Python 3.13)
    runs-on: ubuntu-latest
    timeout-minutes: 45
    env:
      DATA_BOAR_CI_DL: "1"
      HF_HUB_DISABLE_TELEMETRY: "1"
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0
          persist-credentials: false

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.13"

      - name: Install uv
        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
        with:
          version: "0.11.2"

      - name: Install dependencies (dl extra)
        run: uv sync --extra dl --group dev

      - name: Run DL encode smoke
        env:
          HF_HOME: ${{ runner.temp }}/hf-hub
        run: uv run pytest -v -W error --tb=short tests/test_dl_backend_ci.py

  lint:
    name: Lint (pre-commit)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0
          persist-credentials: false

      - name: Install libmariadb-dev (mariadb PyPI connector builds from source on Linux)
        uses: $/.github/actions/install-libmariadb-dev

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"

      - name: Install uv
        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
        with:
          version: "0.11.2"

      - name: Install dependencies
        run: uv sync --extra shares --group dev
      - name: Pre-commit (all files)
        run: uv run pre-commit run --all-files

  bandit:
    name: Bandit (strict)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Install libmariadb-dev (mariadb PyPI connector builds from source on Linux)
        uses: $/.github/actions/install-libmariadb-dev

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"

      - name: Install uv
        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
        with:
          version: "0.11.2"

      - name: Install dependencies
        run: uv sync --extra shares --group dev

      # Strict merge gate: -ll = MEDIUM+ severity, -ii = MEDIUM+ confidence. [tool.bandit] applies (exclude_dirs, skips).
      - name: Run Bandit (Strict Mode)
        run: uv run bandit -r . -c pyproject.toml -ll -ii

  audit:
    name: Dependency audit
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Install libmariadb-dev (mariadb PyPI connector builds from source on Linux)
        uses: $/.github/actions/install-libmariadb-dev

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"

      - name: Install uv
        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
        with:
          version: "0.11.2"

      - name: Install dependencies
        run: uv sync --extra shares

      - name: Install pip-audit
        run: uv pip install pip-audit

      # CVE-2026-3219 cleared: pip>=26.1 fixes (PyPI 26.1.1 as of 2026-05); no longer suppressed.
      # PYSEC-2024-277 / PYSEC-2026-89 / PYSEC-2025-183: OSV entries without PyPI fix versions (2026-05-20); triage via deps issue.
      # PYSEC-2026-217: mariadb 1.1.14 connector — OSV entry without a PyPI fix version (2026-06-17); remove when a fixed release ships. Tracked in #922.
      - name: Run pip audit
        run: uv run pip-audit --ignore-vuln PYSEC-2024-277 --ignore-vuln PYSEC-2026-89 --ignore-vuln PYSEC-2025-183 --ignore-vuln PYSEC-2026-217

  dependency-review:
    name: Dependency review (PR)
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: read
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Dependency Review
        uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0

  # Windows native path (#1427 / PLAN_WINDOWS_CI_ENABLEMENT) — unblocks MSI/winget #1467.
  # MVP: one Python, pytest, pip install smoke, headless --demo equivalent (no dashboard hang).
  # Runs `uv run pytest` (not local check-all). timeout-minutes caps hung runners.
  test-windows:
    name: Test Windows (Python ${{ matrix.python-version }})
    runs-on: windows-latest
    timeout-minutes: 40
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.12"]
    env:
      # Hosted images ship multiple CPythons; without this, `uv sync` may pick
      # 3.13+ despite setup-python installing the matrix version (#1427).
      UV_PYTHON: ${{ matrix.python-version }}
      # Windows runners: OpenMP/joblib multi-thread fits have produced
      # KeyboardInterrupt mid-RandomForest during full pytest (#1427).
      OMP_NUM_THREADS: "1"
      MKL_NUM_THREADS: "1"
      OPENBLAS_NUM_THREADS: "1"
      NUMEXPR_NUM_THREADS: "1"
      JOBLIB_MULTIPROCESSING: "0"
      PYTHONUTF8: "1"
      PYTHONIOENCODING: "utf-8"
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0
          persist-credentials: false

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install uv
        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
        with:
          version: "0.11.2"

      - name: Install dependencies
        # shares: SMB/WebDAV extras (wheels on Windows; no libmariadb apt).
        run: uv sync --python ${{ matrix.python-version }} --extra shares --group dev

      - name: Run tests
        # bash avoids pwsh console CTRL_C quirks seen with sklearn on windows-latest.
        shell: bash
        run: uv run --python ${{ matrix.python-version }} pytest -v -W error --tb=short
        # Same warning posture as Linux test job.

      - name: Native pip install smoke
        shell: pwsh
        run: |
          python -m pip install --upgrade pip
          python -m pip install .
          data-boar --version
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
          # pipx: install into isolated app env (PATH may omit shims on GHA — invoke via pipx run)
          python -m pip install pipx
          pipx run --spec . data-boar --version

      - name: Headless demo smoke (scan + report)
        run: uv run python scripts/demo_headless.py

  sonar:
    name: SonarQube / SonarCloud
    runs-on: ubuntu-latest
    # Do not use `secrets.*` in job-level `if:` — it can prevent the workflow from scheduling any jobs.
    needs: [test]
    permissions:
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0
          persist-credentials: false

      - name: Install libmariadb-dev (mariadb PyPI connector builds from source on Linux)
        uses: $/.github/actions/install-libmariadb-dev

      - name: Detect Sonar token
        id: sonar_token
        env:
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
        run: |
          if [ -n "$SONAR_TOKEN" ]; then
            echo "present=true" >> "$GITHUB_OUTPUT"
          else
            echo "present=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Set up Python
        if: steps.sonar_token.outputs.present == 'true'
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"

      - name: Install uv
        if: steps.sonar_token.outputs.present == 'true'
        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
        with:
          version: "0.11.2"

      - name: Install dependencies
        if: steps.sonar_token.outputs.present == 'true'
        run: uv sync --extra shares

      - name: Download coverage.xml
        if: steps.sonar_token.outputs.present == 'true'
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: coverage-xml

      - name: SonarQube / SonarCloud Scan
        if: steps.sonar_token.outputs.present == 'true'
        uses: SonarSource/sonarqube-scan-action@ba9859eae8dd6bd29e412f25ddbbef3d032000f4 # v8.2.2
        env:
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
          # SonarQube Server only: set SONAR_HOST_URL in repo secrets
          SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}

  # Catch Jinja-invalid Ansible var names / broken role paths before merge (#1631).
  ansible-syntax:
    name: Ansible syntax-check
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"

      - name: Install ansible-core
        run: |
          pip install --disable-pip-version-check --require-hashes \
            -r .github/pip-constraints/ci-ansible-syntax.txt

      - name: Syntax-check LAB-NODE-01 playbooks
        run: bash scripts/ansible-syntax-check.sh

  slack-notify-on-failure:
    name: Slack CI failure notify
    if: ${{ always() && contains(needs.*.result, 'failure') }}
    needs: [test, test-extras, test-dl, test-windows, lint, bandit, audit, dependency-review, sonar, ansible-syntax]
    uses: $/.github/workflows/slack-ci-failure-notify.yml
    with:
      run_name: CI
      head_branch: ${{ github.head_ref || github.ref_name }}
      event: ${{ github.event_name }}
      html_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
    secrets:
      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}