From e79e5689cea8e7fd1ef73184f5a601bd4d784af6 Mon Sep 17 00:00:00 2001 From: Neukz Date: Tue, 16 Jun 2026 01:05:31 +0200 Subject: [PATCH] feat: add dimension tables with SCD Type 2 repositories --- dbt/dbt_project.yml | 7 ++ dbt/models/marts/_marts__models.yml | 45 +++++++++++++ dbt/models/marts/dim_contributors.sql | 77 ++++++++++++++++++++++ dbt/models/marts/dim_dates.sql | 23 +++++++ dbt/models/marts/dim_labels.sql | 32 +++++++++ dbt/models/marts/dim_repositories.sql | 48 ++++++++++++++ dbt/snapshots/snap_repositories.sql | 14 ++++ dbt/tests/assert_dim_date_completeness.sql | 14 ++++ 8 files changed, 260 insertions(+) create mode 100644 dbt/models/marts/_marts__models.yml create mode 100644 dbt/models/marts/dim_contributors.sql create mode 100644 dbt/models/marts/dim_dates.sql create mode 100644 dbt/models/marts/dim_labels.sql create mode 100644 dbt/models/marts/dim_repositories.sql create mode 100644 dbt/snapshots/snap_repositories.sql create mode 100644 dbt/tests/assert_dim_date_completeness.sql diff --git a/dbt/dbt_project.yml b/dbt/dbt_project.yml index c0ef196..db7961b 100644 --- a/dbt/dbt_project.yml +++ b/dbt/dbt_project.yml @@ -22,3 +22,10 @@ models: staging: +materialized: view +schema: staging + marts: + +materialized: table + +schema: marts + +snapshots: + repolytics: + +schema: snapshots diff --git a/dbt/models/marts/_marts__models.yml b/dbt/models/marts/_marts__models.yml new file mode 100644 index 0000000..fd74754 --- /dev/null +++ b/dbt/models/marts/_marts__models.yml @@ -0,0 +1,45 @@ +version: 2 + +models: + - name: dim_dates + description: Calendar dimension, one row per day (2020–2030), generated in SQL. + columns: + - name: date_key + description: Surrogate date key (YYYYMMDD integer). + data_tests: [not_null, unique] + - name: full_date + data_tests: [not_null, unique] + + - name: dim_contributors + description: Type 1 contributor profiles aggregated across commits and pull requests. + columns: + - name: contributor_key + data_tests: [not_null, unique] + - name: username + data_tests: [not_null, unique] + + - name: dim_repositories + description: SCD Type 2 repository dimension (one row per version) built on snap_repositories. + columns: + - name: repository_key + description: Surrogate key, hash of repository_id + the snapshot version timestamp. + data_tests: [not_null, unique] + - name: repository_id + data_tests: [not_null] + - name: is_current + data_tests: [not_null] + data_tests: + - dbt_utils.mutually_exclusive_ranges: + arguments: + lower_bound_column: valid_from + upper_bound_column: valid_to + partition_by: repository_id + gaps: not_allowed + + - name: dim_labels + description: Type 1 dimension of distinct issue/PR labels across all projects. + columns: + - name: label_key + data_tests: [not_null, unique] + - name: label_name + data_tests: [not_null, unique] diff --git a/dbt/models/marts/dim_contributors.sql b/dbt/models/marts/dim_contributors.sql new file mode 100644 index 0000000..309f868 --- /dev/null +++ b/dbt/models/marts/dim_contributors.sql @@ -0,0 +1,77 @@ +-- Type 1 contributor dimension: one row per GitHub login, aggregated across the +-- commit and pull-request staging models. Rebuilt in full each run. + +with commits as ( + select + author_login as username, + repository, + committed_at + from {{ ref('stg_github__commits') }} + where author_login is not null +), + +prs as ( + select + author_login as username, + merged_at + from {{ ref('stg_github__pull_requests') }} + where author_login is not null +), + +contributors as ( + select username from commits + union + select username from prs +), + +commit_stats as ( + select + username, + min(committed_at::date) as first_commit_date, + max(committed_at::date) as last_commit_date, + count(*) as total_commits, + count(distinct repository) as distinct_projects_count + from commits + group by username +), + +pr_stats as ( + select + username, + count(*) as total_prs_opened, + count(*) filter (where merged_at is not null) as total_prs_merged + from prs + group by username +), + +-- Most-committed-to project per contributor (ties broken alphabetically). +primary_project as ( + select username, repository as primary_project + from ( + select + username, + repository, + row_number() over ( + partition by username + order by count(*) desc, repository + ) as rn + from commits + group by username, repository + ) + where rn = 1 +) + +select + {{ dbt_utils.generate_surrogate_key(['c.username']) }} as contributor_key, + c.username, + cs.first_commit_date, + cs.last_commit_date, + coalesce(cs.total_commits, 0) as total_commits, + coalesce(ps.total_prs_opened, 0) as total_prs_opened, + coalesce(ps.total_prs_merged, 0) as total_prs_merged, + coalesce(cs.distinct_projects_count, 0) as distinct_projects_count, + pp.primary_project +from contributors c +left join commit_stats cs on c.username = cs.username +left join pr_stats ps on c.username = ps.username +left join primary_project pp on c.username = pp.username diff --git a/dbt/models/marts/dim_dates.sql b/dbt/models/marts/dim_dates.sql new file mode 100644 index 0000000..f73f5f0 --- /dev/null +++ b/dbt/models/marts/dim_dates.sql @@ -0,0 +1,23 @@ +with spine as ( + {{ + dbt_utils.date_spine( + datepart="day", + start_date="cast('2020-01-01' as date)", + end_date="cast('2031-01-01' as date)" + ) + }} +) + +select + cast(strftime(date_day, '%Y%m%d') as integer) as date_key, + cast(date_day as date) as full_date, + year(date_day) as year, + quarter(date_day) as quarter, + month(date_day) as month, + monthname(date_day) as month_name, + day(date_day) as day_of_month, + dayofweek(date_day) as day_of_week, + dayname(date_day) as day_name, + dayofweek(date_day) in (0, 6) as is_weekend, + week(date_day) as week_of_year +from spine diff --git a/dbt/models/marts/dim_labels.sql b/dbt/models/marts/dim_labels.sql new file mode 100644 index 0000000..5c7c656 --- /dev/null +++ b/dbt/models/marts/dim_labels.sql @@ -0,0 +1,32 @@ +-- Type 1 label dimension: distinct issue/PR labels across all projects. Labels land +-- in staging as a JSON array per row, so we explode the arrays and dedupe by name. + +with labels_raw as ( + select labels + from {{ ref('stg_github__pull_requests') }} + where labels is not null + union all + select labels + from {{ ref('stg_github__issues') }} + where labels is not null +), + +exploded as ( + select unnest(json_extract(labels, '$[*]')) as label + from labels_raw +), + +distinct_labels as ( + select + label ->> '$.name' as label_name, + max(label ->> '$.color') as label_color + from exploded + where (label ->> '$.name') is not null + group by label ->> '$.name' +) + +select + {{ dbt_utils.generate_surrogate_key(['label_name']) }} as label_key, + label_name, + label_color +from distinct_labels diff --git a/dbt/models/marts/dim_repositories.sql b/dbt/models/marts/dim_repositories.sql new file mode 100644 index 0000000..b809e83 --- /dev/null +++ b/dbt/models/marts/dim_repositories.sql @@ -0,0 +1,48 @@ +-- SCD Type 2 repository dimension: a thin presentation model over snap_repositories. +-- The dbt snapshot owns the history (dbt_valid_from/to); here we reshape it into the +-- star-schema contract (surrogate key + valid_from/valid_to/is_current). + +with versioned as ( + select + repository_id, + repository_name, + name, + owner_login, + description, + stars, + forks, + open_issues, + language, + license_spdx, + topics, + created_at, + updated_at, + -- Full-resolution version timestamp drives the surrogate key so it stays + -- unique even if a repo changes more than once on the same calendar day; + -- valid_from/valid_to are kept at date grain for fact date-range joins. + dbt_valid_from as version_ts, + dbt_valid_from::date as valid_from, + coalesce(dbt_valid_to::date, date '9999-12-31') as valid_to, + dbt_valid_to is null as is_current + from {{ ref('snap_repositories') }} +) + +select + {{ dbt_utils.generate_surrogate_key(['repository_id', 'version_ts']) }} as repository_key, + repository_id, + repository_name, + name, + owner_login, + description, + stars, + forks, + open_issues, + language, + license_spdx, + topics, + created_at, + updated_at, + valid_from, + valid_to, + is_current +from versioned diff --git a/dbt/snapshots/snap_repositories.sql b/dbt/snapshots/snap_repositories.sql new file mode 100644 index 0000000..b8b6f8f --- /dev/null +++ b/dbt/snapshots/snap_repositories.sql @@ -0,0 +1,14 @@ +{% snapshot snap_repositories %} + +{{ + config( + unique_key='repository_id', + strategy='check', + check_cols=['stars', 'forks', 'open_issues', 'description', 'topics'], + ) +}} + +-- SCD Type 2 history capture for repositories. +select * from {{ ref('stg_github__repositories') }} + +{% endsnapshot %} diff --git a/dbt/tests/assert_dim_date_completeness.sql b/dbt/tests/assert_dim_date_completeness.sql new file mode 100644 index 0000000..40deaca --- /dev/null +++ b/dbt/tests/assert_dim_date_completeness.sql @@ -0,0 +1,14 @@ +-- Fails if consecutive days in dim_dates are not exactly one day apart, i.e. the +-- generated date dimension has a gap. Passes when zero rows are returned. + +with ordered as ( + select + full_date, + lead(full_date) over (order by full_date) as next_date + from {{ ref('dim_dates') }} +) + +select * +from ordered +where next_date is not null + and next_date <> full_date + 1