feat: add raw loader and dbt staging models with tests

.github/workflows/ci.yml

@@ -25,7 +25,7 @@ jobs:
           cache-dependency-glob: uv.lock
 
       - name: Install dependencies
-        run: uv sync --locked
+        run: uv sync --locked --group dbt
 
       - name: Lint
         run: uv run --no-sync ruff check .
@@ -35,3 +35,14 @@ jobs:
 
       - name: Unit tests
         run: uv run --no-sync pytest tests/unit
+
+      - name: Integration tests
+        run: uv run --no-sync pytest tests/integration
+
+      - name: Install dbt packages
+        working-directory: dbt
+        run: uv run --no-sync dbt deps
+
+      - name: Compile dbt models
+        working-directory: dbt
+        run: uv run --no-sync dbt compile --target ci

.gitignore

@@ -34,6 +34,7 @@ orchestration/logs/
 dbt/target/
 dbt/dbt_packages/
 dbt/logs/
+dbt/.user.yml
 
 # OS / editor
 .DS_Store

Taskfile.yml

@@ -50,3 +50,43 @@ tasks:
       - task: lint
       - task: format:check
       - task: test:unit
+      - task: test:integration
+      - task: dbt:deps
+      - task: dbt:compile
+
+  # ----- Loading -----
+  load:raw:
+    desc: Load landed Parquet into the DuckDB raw schema
+    cmds:
+      - uv run python -c "from repolytics.loading.raw_loader import load_all; print(load_all())"
+
+  # ----- dbt -----
+  dbt:deps:
+    desc: Install dbt packages
+    dir: dbt
+    cmds:
+      - uv run --group dbt dbt deps
+
+  dbt:compile:
+    desc: Compile dbt models (validate SQL without executing)
+    dir: dbt
+    cmds:
+      - uv run --group dbt dbt compile --target ci
+
+  dbt:run:
+    desc: Run dbt models
+    dir: dbt
+    cmds:
+      - uv run --group dbt dbt run
+
+  dbt:test:
+    desc: Run dbt tests
+    dir: dbt
+    cmds:
+      - uv run --group dbt dbt test
+
+  dbt:build:
+    desc: Run dbt models and tests
+    dir: dbt
+    cmds:
+      - uv run --group dbt dbt build

dbt/dbt_project.yml

@@ -0,0 +1,24 @@
+name: 'repolytics'
+version: '1.0.0'
+
+config-version: 2
+profile: 'repolytics'
+
+require-dbt-version: [">=1.11.0", "<2.0.0"]
+
+model-paths: ["models"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+target-path: target
+clean-targets:
+  - "target"
+  - "dbt_packages"
+
+models:
+  repolytics:
+    staging:
+      +materialized: view
+      +schema: staging

dbt/macros/generate_schema_name.sql

@@ -0,0 +1,12 @@
+{#
+    Use the configured custom schema name verbatim (e.g. `staging`, `marts`)
+    instead of dbt default `{target_schema}_{custom}` concatenation, so the
+    warehouse layers map to clean schema names.
+#}
+{% macro generate_schema_name(custom_schema_name, node) -%}
+    {%- if custom_schema_name is none -%}
+        {{ target.schema }}
+    {%- else -%}
+        {{ custom_schema_name | trim }}
+    {%- endif -%}
+{%- endmacro %}

dbt/models/staging/github/_github__models.yml

@@ -0,0 +1,55 @@
+version: 2
+
+models:
+  - name: stg_github__repositories
+    description: One cleaned row per repository snapshot.
+    columns:
+      - name: repository_id
+        description: Natural key from the GitHub API.
+        data_tests: [not_null, unique]
+      - name: repository_name
+        data_tests: [not_null]
+
+  - name: stg_github__commits
+    description: One cleaned row per non-merge commit.
+    columns:
+      - name: commit_sha
+        data_tests: [not_null]
+      - name: repository
+        data_tests: [not_null]
+    data_tests:
+      - dbt_utils.unique_combination_of_columns:
+          arguments:
+            combination_of_columns: [repository, commit_sha]
+
+  - name: stg_github__pull_requests
+    description: One cleaned row per pull request.
+    columns:
+      - name: pr_number
+        data_tests: [not_null]
+      - name: repository
+        data_tests: [not_null]
+    data_tests:
+      - dbt_utils.unique_combination_of_columns:
+          arguments:
+            combination_of_columns: [repository, pr_number]
+
+  - name: stg_github__issues
+    description: One cleaned row per issue (pull requests excluded).
+    columns:
+      - name: issue_number
+        data_tests: [not_null]
+      - name: repository
+        data_tests: [not_null]
+    data_tests:
+      - dbt_utils.unique_combination_of_columns:
+          arguments:
+            combination_of_columns: [repository, issue_number]
+
+  - name: stg_github__releases
+    description: One cleaned row per release.
+    columns:
+      - name: release_id
+        data_tests: [not_null, unique]
+      - name: repository
+        data_tests: [not_null]

dbt/models/staging/github/_github__sources.yml

@@ -0,0 +1,21 @@
+version: 2
+
+sources:
+  - name: github
+    description: Raw GitHub API responses, landed as a JSON `data` blob per record.
+    schema: raw
+    loaded_at_field: _loaded_at
+    freshness:
+      warn_after: {count: 12, period: hour}
+      error_after: {count: 24, period: hour}
+    tables:
+      - name: repositories
+        description: One row per repository snapshot.
+      - name: commits
+        description: One row per commit; `_repo` carries the source repository.
+      - name: pull_requests
+        description: One row per pull request; `_repo` carries the source repository.
+      - name: issues
+        description: One row per issue (includes PRs upstream; staging filters them).
+      - name: releases
+        description: One row per release; `_repo` carries the source repository.

dbt/models/staging/github/stg_github__commits.sql

@@ -0,0 +1,19 @@
+with source as (
+    select data::json as d, _repo, _loaded_at
+    from {{ source('github', 'commits') }}
+)
+
+select
+    d ->> '$.sha' as commit_sha,
+    _repo as repository,
+    d ->> '$.author.login' as author_login,
+    d ->> '$.commit.author.name' as author_name,
+    d ->> '$.commit.author.email' as author_email,
+    (d ->> '$.commit.author.date')::timestamp as committed_at,
+    (d ->> '$.stats.additions')::bigint as additions,
+    (d ->> '$.stats.deletions')::bigint as deletions,
+    d ->> '$.commit.message' as message,
+    _loaded_at
+from source
+-- Drop merge commits (more than one parent); rows without `parents` are kept.
+where coalesce(json_array_length(d -> '$.parents'), 0) <= 1

dbt/models/staging/github/stg_github__issues.sql

@@ -0,0 +1,26 @@
+with source as (
+    select data::json as d, _repo, _loaded_at
+    from {{ source('github', 'issues') }}
+)
+
+select
+    _repo as repository,
+    (d ->> '$.number')::bigint as issue_number,
+    d ->> '$.user.login' as author_login,
+    d ->> '$.state' as state,
+    (d ->> '$.state') = 'closed' as is_closed,
+    (d ->> '$.created_at')::timestamp as created_at,
+    (d ->> '$.closed_at')::timestamp as closed_at,
+    case
+        when (d ->> '$.closed_at') is not null then datediff(
+            'hour',
+            (d ->> '$.created_at')::timestamp,
+            (d ->> '$.closed_at')::timestamp
+        )
+    end as time_to_close_hours,
+    (d ->> '$.comments')::bigint as comment_count,
+    d -> '$.labels' as labels,
+    _loaded_at
+from source
+-- The issues endpoint returns PRs too; drop them (real issues have no `pull_request`).
+where (d -> '$.pull_request') is null

dbt/models/staging/github/stg_github__pull_requests.sql

@@ -0,0 +1,26 @@
+with source as (
+    select data::json as d, _repo, _loaded_at
+    from {{ source('github', 'pull_requests') }}
+)
+
+select
+    _repo as repository,
+    (d ->> '$.number')::bigint as pr_number,
+    d ->> '$.user.login' as author_login,
+    d ->> '$.state' as state,
+    (d ->> '$.created_at')::timestamp as created_at,
+    (d ->> '$.merged_at')::timestamp as merged_at,
+    (d ->> '$.additions')::bigint as additions,
+    (d ->> '$.deletions')::bigint as deletions,
+    (d ->> '$.review_comments')::bigint as review_comments,
+    (d ->> '$.comments')::bigint as comment_count,
+    case
+        when (d ->> '$.merged_at') is not null then datediff(
+            'hour',
+            (d ->> '$.created_at')::timestamp,
+            (d ->> '$.merged_at')::timestamp
+        )
+    end as time_to_merge_hours,
+    d -> '$.labels' as labels,
+    _loaded_at
+from source

dbt/models/staging/github/stg_github__releases.sql

@@ -0,0 +1,13 @@
+with source as (
+    select data::json as d, _repo, _loaded_at
+    from {{ source('github', 'releases') }}
+)
+
+select
+    (d ->> '$.id')::bigint as release_id,
+    _repo as repository,
+    d ->> '$.tag_name' as tag_name,
+    d ->> '$.name' as name,
+    (d ->> '$.published_at')::timestamp as published_at,
+    _loaded_at
+from source

dbt/models/staging/github/stg_github__repositories.sql

@@ -0,0 +1,21 @@
+with source as (
+    select data::json as d, _loaded_at
+    from {{ source('github', 'repositories') }}
+)
+
+select
+    (d ->> '$.id')::bigint as repository_id,
+    d ->> '$.full_name' as repository_name,
+    d ->> '$.name' as name,
+    d ->> '$.owner.login' as owner_login,
+    d ->> '$.description' as description,
+    (d ->> '$.stargazers_count')::bigint as stars,
+    (d ->> '$.forks_count')::bigint as forks,
+    (d ->> '$.open_issues_count')::bigint as open_issues,
+    d ->> '$.language' as language,
+    d ->> '$.license.spdx_id' as license_spdx,
+    d -> '$.topics' as topics,
+    (d ->> '$.created_at')::timestamp as created_at,
+    (d ->> '$.updated_at')::timestamp as updated_at,
+    _loaded_at
+from source

dbt/models/staging/pypi/_pypi__models.yml

@@ -0,0 +1,16 @@
+version: 2
+
+models:
+  - name: stg_pypi__downloads
+    description: One cleaned row per package/category/day download count.
+    columns:
+      - name: package
+        data_tests: [not_null]
+      - name: download_date
+        data_tests: [not_null]
+      - name: download_count
+        data_tests: [not_null]
+    data_tests:
+      - dbt_utils.unique_combination_of_columns:
+          arguments:
+            combination_of_columns: [package, category, download_date]

dbt/models/staging/pypi/_pypi__sources.yml

@@ -0,0 +1,13 @@
+version: 2
+
+sources:
+  - name: pypi
+    description: Raw PyPI Stats responses; one row per package/day/category.
+    schema: raw
+    loaded_at_field: _loaded_at
+    freshness:
+      warn_after: {count: 12, period: hour}
+      error_after: {count: 48, period: hour}
+    tables:
+      - name: downloads
+        description: Daily download counts; `_package` carries the source package.

dbt/models/staging/pypi/stg_pypi__downloads.sql

@@ -0,0 +1,12 @@
+with source as (
+    select data::json as d, _package, _loaded_at
+    from {{ source('pypi', 'downloads') }}
+)
+
+select
+    _package as package,
+    d ->> '$.category' as category,
+    (d ->> '$.date')::date as download_date,
+    (d ->> '$.downloads')::bigint as download_count,
+    _loaded_at
+from source

dbt/package-lock.yml

@@ -0,0 +1,5 @@
+packages:
+  - name: dbt_utils
+    package: dbt-labs/dbt_utils
+    version: 1.3.3
+sha1_hash: a4da77dcded39caf20bd661f0098cbffd9735800

dbt/packages.yml

@@ -0,0 +1,3 @@
+packages:
+  - package: dbt-labs/dbt_utils
+    version: [">=1.3.0", "<2.0.0"]

dbt/profiles.yml

@@ -0,0 +1,12 @@
+repolytics:
+  target: dev
+  outputs:
+    dev:
+      type: duckdb
+      path: "{{ env_var('DUCKDB_PATH', '../data/warehouse/repolytics.duckdb') }}"
+      threads: 4
+    ci:
+      # Ephemeral in-memory DB: enough for `dbt compile` / parsing, no data needed.
+      type: duckdb
+      path: ":memory:"
+      threads: 4