ci: execute dbt in CI and split into parallel jobs

.github/workflows/ci.yml

@@ -9,40 +9,109 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
-  check:
+  changes:
     runs-on: ubuntu-latest
     permissions:
-      contents: read
+      pull-requests: read
+    outputs:
+      docker: ${{ steps.filter.outputs.docker }}
     steps:
-      - name: Checkout
-        uses: actions/checkout@v6
+      - uses: dorny/paths-filter@v4
+        id: filter
+        with:
+          filters: |
+            docker:
+              - 'Dockerfile'
+              - 'pyproject.toml'
+              - 'uv.lock'
 
-      - name: Install uv
-        uses: astral-sh/setup-uv@v7
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@v7
         with:
           enable-cache: true
           cache-dependency-glob: uv.lock
-
       - name: Install dependencies
-        run: uv sync --locked --group dbt
-
+        run: uv sync --locked --group dev
       - name: Lint
         run: uv run --no-sync ruff check .
-
-      - name: Format check
+      - name: Check formatting
         run: uv run --no-sync ruff format --check .
 
-      - name: Unit tests
-        run: uv run --no-sync pytest tests/unit
-
-      - name: Integration tests
-        run: uv run --no-sync pytest tests/integration
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+      - name: Install dependencies
+        run: uv sync --locked --group dev
+      - name: Unit + integration tests + coverage
+        run: uv run --no-sync pytest --cov --cov-report=term-missing
 
+  dbt-build:
+    runs-on: ubuntu-latest
+    env:
+      DUCKDB_PATH: ${{ github.workspace }}/ci.duckdb
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+      - name: Install dependencies
+        run: uv sync --locked --group dbt --group dev
+      - name: Seed raw schema
+        run: uv run --no-sync python -m tests.support.warehouse
       - name: Install dbt packages
         working-directory: dbt
         run: uv run --no-sync dbt deps
+      - name: dbt build (run models + tests on fixture data)
+        working-directory: dbt
+        run: uv run --no-sync dbt build --target ci
 
-      - name: Compile dbt models
+  dag-validate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+      - name: Install dependencies
+        run: uv sync --locked --all-groups
+      - name: Install dbt packages
+        working-directory: dbt
+        run: uv run --no-sync dbt deps
+      - name: Generate dbt manifest
         working-directory: dbt
-        run: uv run --no-sync dbt compile --target ci
+        run: uv run --no-sync dbt parse
+      - name: DAG tests
+        run: uv run --no-sync pytest tests/dag
+
+  docker-build:
+    needs: changes
+    if: needs.changes.outputs.docker == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - name: Build Airflow image
+        run: docker build .

Taskfile.yml

@@ -4,13 +4,13 @@ version: '3'
 
 tasks:
   install:
-    desc: Install all dependencies with uv
+    desc: Install all dependencies
     cmds:
       - uv sync --all-groups
 
   # ----- Quality: lint / format / test -----
   lint:
-    desc: Run ruff linter
+    desc: Run ruff linter (without fixes)
     cmds:
       - uv run ruff check .
 
@@ -20,19 +20,21 @@ tasks:
       - uv run ruff check --fix .
 
   format:
-    desc: Run ruff formatter (modifies files)
+    desc: Run ruff formatter (applies fixes)
     cmds:
       - uv run ruff format .
 
   format:check:
-    desc: Check formatting without modifying
+    desc: Run ruff linter (without fixes)
     cmds:
       - uv run ruff format --check .
 
   test:
-    desc: Run all tests
+    desc: Run all tests (unit, integration, dag)
     cmds:
-      - uv run pytest
+      - task: test:unit
+      - task: test:integration
+      - task: test:dag
 
   test:unit:
     desc: Run unit tests only
@@ -44,20 +46,25 @@ tasks:
     cmds:
       - uv run pytest tests/integration
 
+  test:dag:
+    desc: Run Airflow/Cosmos DAG tests
+    deps: [dbt:manifest]
+    cmds:
+      - uv run --group dbt --group orchestration pytest tests/dag
+
   test:coverage:
-    desc: Run tests with coverage report
+    desc: Run unit + integration tests with coverage report
     cmds:
       - uv run pytest --cov --cov-report=term-missing
 
   ci:
-    desc: Run full CI check locally (mirrors GitHub Actions)
+    desc: Run full CI check locally (mirrors GitHub Actions, excludes Docker build)
     cmds:
       - task: lint
       - task: format:check
-      - task: test:unit
-      - task: test:integration
-      - task: dbt:deps
-      - task: dbt:compile
+      - task: test:coverage
+      - task: dbt:build:ci
+      - task: test:dag
 
   # ----- dlt -----
   dlt:ingest:
@@ -92,7 +99,7 @@ tasks:
     desc: Compile dbt models (validate SQL without executing)
     dir: dbt
     cmds:
-      - uv run --group dbt dbt compile --target ci
+      - uv run --group dbt dbt compile
 
   dbt:manifest:
     desc: Generate target/manifest.json for Cosmos (DBT_MANIFEST load mode)
@@ -118,16 +125,18 @@ tasks:
     cmds:
       - uv run --group dbt dbt build
 
+  dbt:build:ci:
+    desc: Seed raw from fixtures and run dbt build against a throwaway DuckDB (mirrors CI)
+    env:
+      DUCKDB_PATH: /tmp/repolytics-ci.duckdb
+    cmds:
+      - rm -f "$DUCKDB_PATH"
+      - uv run python -m tests.support.warehouse
+      - cd dbt && uv run --group dbt dbt deps && uv run --group dbt dbt build --target ci
+
   dbt:docs:
     desc: Generate and serve dbt documentation
     dir: dbt
     cmds:
       - uv run --group dbt dbt docs generate
       - uv run --group dbt dbt docs serve
-
-  # ----- Docker -----
-  docker:build:
-    desc: Build the custom Airflow image
-    deps: [dbt:manifest]
-    cmds:
-      - docker compose build

dbt/profiles.yml

@@ -6,7 +6,8 @@ repolytics:
       path: "{{ env_var('DUCKDB_PATH', '../data/warehouse/repolytics.duckdb') }}"
       threads: 4
     ci:
-      # Ephemeral in-memory DB: enough for `dbt compile` / parsing, no data needed.
+      # File-backed DuckDB seeded from fixtures (tests/support/warehouse.py).
+      # Lets CI run `dbt build` (models + tests). DUCKDB_PATH is set by the CI job.
       type: duckdb
-      path: ":memory:"
+      path: "{{ env_var('DUCKDB_PATH') }}"
       threads: 4

pyproject.toml

@@ -37,7 +37,7 @@ line-length = 88
 select = ["E", "W", "F", "I", "UP", "B", "C4", "SIM", "PTH", "RUF"]
 
 [tool.pytest.ini_options]
-testpaths = ["tests"]
+testpaths = ["tests/unit", "tests/integration"]
 addopts = "-ra --strict-markers --strict-config"
 xfail_strict = true