diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f0a113..af9f67f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,40 +9,109 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: - check: + changes: runs-on: ubuntu-latest permissions: - contents: read + pull-requests: read + outputs: + docker: ${{ steps.filter.outputs.docker }} steps: - - name: Checkout - uses: actions/checkout@v6 + - uses: dorny/paths-filter@v4 + id: filter + with: + filters: | + docker: + - 'Dockerfile' + - 'pyproject.toml' + - 'uv.lock' - - name: Install uv - uses: astral-sh/setup-uv@v7 + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@v7 with: enable-cache: true cache-dependency-glob: uv.lock - - name: Install dependencies - run: uv sync --locked --group dbt - + run: uv sync --locked --group dev - name: Lint run: uv run --no-sync ruff check . - - - name: Format check + - name: Check formatting run: uv run --no-sync ruff format --check . - - name: Unit tests - run: uv run --no-sync pytest tests/unit - - - name: Integration tests - run: uv run --no-sync pytest tests/integration + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + cache-dependency-glob: uv.lock + - name: Install dependencies + run: uv sync --locked --group dev + - name: Unit + integration tests + coverage + run: uv run --no-sync pytest --cov --cov-report=term-missing + dbt-build: + runs-on: ubuntu-latest + env: + DUCKDB_PATH: ${{ github.workspace }}/ci.duckdb + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + cache-dependency-glob: uv.lock + - name: Install dependencies + run: uv sync --locked --group dbt --group dev + - name: Seed raw schema + run: uv run --no-sync python -m tests.support.warehouse - name: Install dbt packages working-directory: dbt run: uv run --no-sync dbt deps + - name: dbt build (run models + tests on fixture data) + working-directory: dbt + run: uv run --no-sync dbt build --target ci - - name: Compile dbt models + dag-validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + cache-dependency-glob: uv.lock + - name: Install dependencies + run: uv sync --locked --all-groups + - name: Install dbt packages + working-directory: dbt + run: uv run --no-sync dbt deps + - name: Generate dbt manifest working-directory: dbt - run: uv run --no-sync dbt compile --target ci + run: uv run --no-sync dbt parse + - name: DAG tests + run: uv run --no-sync pytest tests/dag + + docker-build: + needs: changes + if: needs.changes.outputs.docker == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Build Airflow image + run: docker build . diff --git a/Taskfile.yml b/Taskfile.yml index 32af86b..698fb61 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -4,13 +4,13 @@ version: '3' tasks: install: - desc: Install all dependencies with uv + desc: Install all dependencies cmds: - uv sync --all-groups # ----- Quality: lint / format / test ----- lint: - desc: Run ruff linter + desc: Run ruff linter (without fixes) cmds: - uv run ruff check . @@ -20,19 +20,21 @@ tasks: - uv run ruff check --fix . format: - desc: Run ruff formatter (modifies files) + desc: Run ruff formatter (applies fixes) cmds: - uv run ruff format . format:check: - desc: Check formatting without modifying + desc: Run ruff linter (without fixes) cmds: - uv run ruff format --check . test: - desc: Run all tests + desc: Run all tests (unit, integration, dag) cmds: - - uv run pytest + - task: test:unit + - task: test:integration + - task: test:dag test:unit: desc: Run unit tests only @@ -44,20 +46,25 @@ tasks: cmds: - uv run pytest tests/integration + test:dag: + desc: Run Airflow/Cosmos DAG tests + deps: [dbt:manifest] + cmds: + - uv run --group dbt --group orchestration pytest tests/dag + test:coverage: - desc: Run tests with coverage report + desc: Run unit + integration tests with coverage report cmds: - uv run pytest --cov --cov-report=term-missing ci: - desc: Run full CI check locally (mirrors GitHub Actions) + desc: Run full CI check locally (mirrors GitHub Actions, excludes Docker build) cmds: - task: lint - task: format:check - - task: test:unit - - task: test:integration - - task: dbt:deps - - task: dbt:compile + - task: test:coverage + - task: dbt:build:ci + - task: test:dag # ----- dlt ----- dlt:ingest: @@ -92,7 +99,7 @@ tasks: desc: Compile dbt models (validate SQL without executing) dir: dbt cmds: - - uv run --group dbt dbt compile --target ci + - uv run --group dbt dbt compile dbt:manifest: desc: Generate target/manifest.json for Cosmos (DBT_MANIFEST load mode) @@ -118,16 +125,18 @@ tasks: cmds: - uv run --group dbt dbt build + dbt:build:ci: + desc: Seed raw from fixtures and run dbt build against a throwaway DuckDB (mirrors CI) + env: + DUCKDB_PATH: /tmp/repolytics-ci.duckdb + cmds: + - rm -f "$DUCKDB_PATH" + - uv run python -m tests.support.warehouse + - cd dbt && uv run --group dbt dbt deps && uv run --group dbt dbt build --target ci + dbt:docs: desc: Generate and serve dbt documentation dir: dbt cmds: - uv run --group dbt dbt docs generate - uv run --group dbt dbt docs serve - - # ----- Docker ----- - docker:build: - desc: Build the custom Airflow image - deps: [dbt:manifest] - cmds: - - docker compose build diff --git a/dbt/profiles.yml b/dbt/profiles.yml index 8e16857..42982f3 100644 --- a/dbt/profiles.yml +++ b/dbt/profiles.yml @@ -6,7 +6,8 @@ repolytics: path: "{{ env_var('DUCKDB_PATH', '../data/warehouse/repolytics.duckdb') }}" threads: 4 ci: - # Ephemeral in-memory DB: enough for `dbt compile` / parsing, no data needed. + # File-backed DuckDB seeded from fixtures (tests/support/warehouse.py). + # Lets CI run `dbt build` (models + tests). DUCKDB_PATH is set by the CI job. type: duckdb - path: ":memory:" + path: "{{ env_var('DUCKDB_PATH') }}" threads: 4 diff --git a/docker-compose.yaml b/docker-compose.yml similarity index 100% rename from docker-compose.yaml rename to docker-compose.yml diff --git a/pyproject.toml b/pyproject.toml index a226bc6..ac571d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ line-length = 88 select = ["E", "W", "F", "I", "UP", "B", "C4", "SIM", "PTH", "RUF"] [tool.pytest.ini_options] -testpaths = ["tests"] +testpaths = ["tests/unit", "tests/integration"] addopts = "-ra --strict-markers --strict-config" xfail_strict = true