diff --git a/dbt/models/marts/_marts__models.yml b/dbt/models/marts/_marts__models.yml index fd74754..ea0eb88 100644 --- a/dbt/models/marts/_marts__models.yml +++ b/dbt/models/marts/_marts__models.yml @@ -2,7 +2,7 @@ version: 2 models: - name: dim_dates - description: Calendar dimension, one row per day (2020–2030), generated in SQL. + description: Calendar dimension, one row per day (2020-2030), generated in SQL. columns: - name: date_key description: Surrogate date key (YYYYMMDD integer). @@ -43,3 +43,162 @@ models: data_tests: [not_null, unique] - name: label_name data_tests: [not_null, unique] + + - name: fct_commits + description: One row per commit. Repository resolved via the SCD2 half-open date range. + columns: + - name: commit_key + description: Surrogate key, hash of repository + commit_sha. + data_tests: [not_null, unique] + - name: repository_key + data_tests: + - not_null + - relationships: + arguments: + to: ref('dim_repositories') + field: repository_key + - name: contributor_key + description: Nullable - commits whose author has no GitHub login stay unattributed. + data_tests: + - relationships: + arguments: + to: ref('dim_contributors') + field: contributor_key + - name: date_key + data_tests: + - not_null + - relationships: + arguments: + to: ref('dim_dates') + field: date_key + - name: lines_added + data_tests: + - dbt_utils.expression_is_true: + arguments: + expression: ">= 0" + - name: lines_deleted + data_tests: + - dbt_utils.expression_is_true: + arguments: + expression: ">= 0" + + - name: fct_pull_requests + description: One row per pull request. Repository resolved via the SCD2 half-open range on the PR open date. + columns: + - name: pr_key + description: Surrogate key, hash of repository + pr_number. + data_tests: [not_null, unique] + - name: repository_key + data_tests: + - not_null + - relationships: + arguments: + to: ref('dim_repositories') + field: repository_key + - name: author_key + description: Nullable FK to dim_contributors. + data_tests: + - relationships: + arguments: + to: ref('dim_contributors') + field: contributor_key + - name: opened_date_key + data_tests: + - not_null + - relationships: + arguments: + to: ref('dim_dates') + field: date_key + - name: merged_date_key + description: Nullable - only set for merged PRs. + data_tests: + - relationships: + arguments: + to: ref('dim_dates') + field: date_key + - name: review_count + data_tests: + - dbt_utils.expression_is_true: + arguments: + expression: ">= 0" + + - name: fct_issues + description: One row per issue. Repository resolved via the SCD2 half-open range. + columns: + - name: issue_key + description: Surrogate key, hash of repository + issue_number. + data_tests: [not_null, unique] + - name: repository_key + data_tests: + - not_null + - relationships: + arguments: + to: ref('dim_repositories') + field: repository_key + - name: author_key + description: Nullable FK to dim_contributors. + data_tests: + - relationships: + arguments: + to: ref('dim_contributors') + field: contributor_key + - name: opened_date_key + data_tests: + - not_null + - relationships: + arguments: + to: ref('dim_dates') + field: date_key + - name: closed_date_key + description: Nullable - only set for closed issues. + data_tests: + - relationships: + arguments: + to: ref('dim_dates') + field: date_key + - name: comment_count + data_tests: + - dbt_utils.expression_is_true: + arguments: + expression: ">= 0" + + - name: bridge_issue_labels + description: Bridge for the issue<->label many-to-many. Grain - one row per issue/label. + data_tests: + - dbt_utils.unique_combination_of_columns: + arguments: + combination_of_columns: [issue_key, label_key] + columns: + - name: issue_key + data_tests: + - not_null + - relationships: + arguments: + to: ref('fct_issues') + field: issue_key + - name: label_key + data_tests: + - not_null + - relationships: + arguments: + to: ref('dim_labels') + field: label_key + + - name: fct_daily_downloads + description: One row per package per day (PyPI 'without_mirrors' downloads). + columns: + - name: download_key + description: Surrogate key, hash of package + download_date. + data_tests: [not_null, unique] + - name: date_key + data_tests: + - not_null + - relationships: + arguments: + to: ref('dim_dates') + field: date_key + - name: download_count + data_tests: + - dbt_utils.expression_is_true: + arguments: + expression: ">= 0" diff --git a/dbt/models/marts/bridge_issue_labels.sql b/dbt/models/marts/bridge_issue_labels.sql new file mode 100644 index 0000000..e24aded --- /dev/null +++ b/dbt/models/marts/bridge_issue_labels.sql @@ -0,0 +1,27 @@ +-- Bridge table for the issue<->label many-to-many. Labels land in staging as a JSON +-- array per issue; we explode them, recompute the issue surrogate key the same way as +-- fct_issues, and resolve each label name to its label_key in dim_labels. +-- Grain: one row per (issue, label). + +with issue_labels as ( + select + repository, + issue_number, + unnest(json_extract(labels, '$[*]')) as label + from {{ ref('stg_github__issues') }} + where labels is not null +), + +exploded as ( + select + {{ dbt_utils.generate_surrogate_key(['repository', 'issue_number']) }} as issue_key, + label ->> '$.name' as label_name + from issue_labels + where (label ->> '$.name') is not null +) + +select distinct + e.issue_key, + dl.label_key +from exploded e +inner join {{ ref('dim_labels') }} dl on e.label_name = dl.label_name diff --git a/dbt/models/marts/dim_repositories.sql b/dbt/models/marts/dim_repositories.sql index b809e83..57f3a96 100644 --- a/dbt/models/marts/dim_repositories.sql +++ b/dbt/models/marts/dim_repositories.sql @@ -21,7 +21,16 @@ with versioned as ( -- unique even if a repo changes more than once on the same calendar day; -- valid_from/valid_to are kept at date grain for fact date-range joins. dbt_valid_from as version_ts, - dbt_valid_from::date as valid_from, + -- The earliest version opens at a past-infinity sentinel ('1900-01-01') + -- so historical facts that predate the first snapshot still resolve to + -- the first captured version instead of dropping. + case + when row_number() over ( + partition by repository_id order by dbt_valid_from + ) = 1 then date '1900-01-01' + else dbt_valid_from::date + end as valid_from, + -- '9999-12-31' = future-infinity bound coalesce(dbt_valid_to::date, date '9999-12-31') as valid_to, dbt_valid_to is null as is_current from {{ ref('snap_repositories') }} diff --git a/dbt/models/marts/fct_commits.sql b/dbt/models/marts/fct_commits.sql new file mode 100644 index 0000000..a2eac83 --- /dev/null +++ b/dbt/models/marts/fct_commits.sql @@ -0,0 +1,25 @@ +-- Commit activity fact: one row per commit. Joins to dim_repositories via the SCD2 +-- half-open date range (event_date >= valid_from AND < valid_to) so each commit maps +-- to the repository version that was current when it landed; to dim_contributors on +-- author login; and to dim_dates via an inline YYYYMMDD key. + +with commits as ( + select * from {{ ref('stg_github__commits') }} +) + +select + {{ dbt_utils.generate_surrogate_key(['c.repository', 'c.commit_sha']) }} as commit_key, + r.repository_key, + co.contributor_key, + cast(strftime(c.committed_at, '%Y%m%d') as integer) as date_key, + c.commit_sha as commit_hash, + c.additions as lines_added, + c.deletions as lines_deleted, + c.committed_at +from commits c +left join {{ ref('dim_repositories') }} r + on c.repository = r.repository_name + and c.committed_at::date >= r.valid_from + and c.committed_at::date < r.valid_to +left join {{ ref('dim_contributors') }} co + on c.author_login = co.username diff --git a/dbt/models/marts/fct_daily_downloads.sql b/dbt/models/marts/fct_daily_downloads.sql new file mode 100644 index 0000000..c5c4090 --- /dev/null +++ b/dbt/models/marts/fct_daily_downloads.sql @@ -0,0 +1,15 @@ +-- PyPI daily download fact: one row per package per day. +-- Filtered to the 'without_mirrors' overall time-series category to avoid double +-- counting 'with_mirrors' and to exclude the recent-endpoint 'last_*' aggregates. + +with downloads as ( + select * from {{ ref('stg_pypi__downloads') }} + where category = 'without_mirrors' +) + +select + {{ dbt_utils.generate_surrogate_key(['package', 'download_date']) }} as download_key, + package, + cast(strftime(download_date, '%Y%m%d') as integer) as date_key, + download_count +from downloads diff --git a/dbt/models/marts/fct_issues.sql b/dbt/models/marts/fct_issues.sql new file mode 100644 index 0000000..9a43bf7 --- /dev/null +++ b/dbt/models/marts/fct_issues.sql @@ -0,0 +1,24 @@ +-- Issue lifecycle fact: one row per issue. Repository resolved via the SCD2 half-open +-- range on the issue open date; author via dim_contributors; opened/closed date keys +-- are inline YYYYMMDD references to dim_dates (closed is nullable). + +with issues as ( + select * from {{ ref('stg_github__issues') }} +) + +select + {{ dbt_utils.generate_surrogate_key(['i.repository', 'i.issue_number']) }} as issue_key, + r.repository_key, + co.contributor_key as author_key, + cast(strftime(i.created_at, '%Y%m%d') as integer) as opened_date_key, + cast(strftime(i.closed_at, '%Y%m%d') as integer) as closed_date_key, + i.is_closed, + i.time_to_close_hours, + i.comment_count +from issues i +left join {{ ref('dim_repositories') }} r + on i.repository = r.repository_name + and i.created_at::date >= r.valid_from + and i.created_at::date < r.valid_to +left join {{ ref('dim_contributors') }} co + on i.author_login = co.username diff --git a/dbt/models/marts/fct_pull_requests.sql b/dbt/models/marts/fct_pull_requests.sql new file mode 100644 index 0000000..fa8b464 --- /dev/null +++ b/dbt/models/marts/fct_pull_requests.sql @@ -0,0 +1,27 @@ +-- Pull request lifecycle fact: one row per PR. Repository resolved via the SCD2 +-- half-open range on the PR open date; author via dim_contributors; opened/merged +-- date keys are inline YYYYMMDD references to dim_dates (merged is nullable). + +with pull_requests as ( + select * from {{ ref('stg_github__pull_requests') }} +) + +select + {{ dbt_utils.generate_surrogate_key(['p.repository', 'p.pr_number']) }} as pr_key, + r.repository_key, + co.contributor_key as author_key, + cast(strftime(p.created_at, '%Y%m%d') as integer) as opened_date_key, + cast(strftime(p.merged_at, '%Y%m%d') as integer) as merged_date_key, + p.merged_at is not null as is_merged, + p.time_to_merge_hours, + p.review_comments as review_count, + p.comment_count, + p.additions, + p.deletions +from pull_requests p +left join {{ ref('dim_repositories') }} r + on p.repository = r.repository_name + and p.created_at::date >= r.valid_from + and p.created_at::date < r.valid_to +left join {{ ref('dim_contributors') }} co + on p.author_login = co.username