diff --git a/Taskfile.yml b/Taskfile.yml index 57556db..e59748b 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -90,3 +90,10 @@ tasks: dir: dbt cmds: - uv run --group dbt dbt build + + dbt:docs: + desc: Generate and serve dbt documentation + dir: dbt + cmds: + - uv run --group dbt dbt docs generate + - uv run --group dbt dbt docs serve diff --git a/dbt/dbt_project.yml b/dbt/dbt_project.yml index db7961b..36f443f 100644 --- a/dbt/dbt_project.yml +++ b/dbt/dbt_project.yml @@ -22,6 +22,9 @@ models: staging: +materialized: view +schema: staging + intermediate: + +materialized: view + +schema: intermediate marts: +materialized: table +schema: marts diff --git a/dbt/models/intermediate/_int__models.yml b/dbt/models/intermediate/_int__models.yml new file mode 100644 index 0000000..13245d4 --- /dev/null +++ b/dbt/models/intermediate/_int__models.yml @@ -0,0 +1,30 @@ +version: 2 + +models: + - name: int_contributor_activity + description: > + Tidy contributor activity event stream at username grain - one row per discrete + event (commit, pr_opened, pr_merged, issue_opened, issue_closed). Reusable + building block for monthly aggregation and PR/health metrics. + columns: + - name: username + description: GitHub login of the actor. + data_tests: [not_null] + - name: event_date + description: Date the activity occurred. + data_tests: [not_null] + - name: event_month + description: First day of the month of event_date. + data_tests: [not_null] + - name: activity_type + description: One of commit, pr_opened, pr_merged, issue_opened, issue_closed. + data_tests: + - not_null + - accepted_values: + arguments: + values: + - commit + - pr_opened + - pr_merged + - issue_opened + - issue_closed diff --git a/dbt/models/intermediate/int_contributor_activity.sql b/dbt/models/intermediate/int_contributor_activity.sql new file mode 100644 index 0000000..cb988d0 --- /dev/null +++ b/dbt/models/intermediate/int_contributor_activity.sql @@ -0,0 +1,68 @@ +-- Tidy contributor activity event stream at username grain: one row per discrete +-- activity event. A reusable building block for monthly aggregation (and future PR / +-- health metrics). Each PR and issue contributes up to two events (opened, and +-- merged/closed) so each lands in the month it actually happened. + +with commits as ( + select + author_login as username, + committed_at::date as event_date, + 'commit' as activity_type + from {{ ref('stg_github__commits') }} + where author_login is not null +), + +prs_opened as ( + select + author_login as username, + created_at::date as event_date, + 'pr_opened' as activity_type + from {{ ref('stg_github__pull_requests') }} + where author_login is not null +), + +prs_merged as ( + select + author_login as username, + merged_at::date as event_date, + 'pr_merged' as activity_type + from {{ ref('stg_github__pull_requests') }} + where author_login is not null and merged_at is not null +), + +issues_opened as ( + select + author_login as username, + created_at::date as event_date, + 'issue_opened' as activity_type + from {{ ref('stg_github__issues') }} + where author_login is not null +), + +issues_closed as ( + select + author_login as username, + closed_at::date as event_date, + 'issue_closed' as activity_type + from {{ ref('stg_github__issues') }} + where author_login is not null and closed_at is not null +), + +events as ( + select * from commits + union all + select * from prs_opened + union all + select * from prs_merged + union all + select * from issues_opened + union all + select * from issues_closed +) + +select + username, + event_date, + date_trunc('month', event_date) as event_month, + activity_type +from events diff --git a/dbt/models/marts/_marts__models.yml b/dbt/models/marts/_marts__models.yml index ea0eb88..08cc882 100644 --- a/dbt/models/marts/_marts__models.yml +++ b/dbt/models/marts/_marts__models.yml @@ -185,12 +185,18 @@ models: field: label_key - name: fct_daily_downloads - description: One row per package per day (PyPI 'without_mirrors' downloads). + description: > + One row per package per day (PyPI 'without_mirrors' downloads). `package` is a + degenerate dimension - no package->repository mapping exists yet. Incremental + (delete+insert on download_key). columns: - name: download_key description: Surrogate key, hash of package + download_date. data_tests: [not_null, unique] + - name: package + description: PyPI package name (degenerate dimension). - name: date_key + description: FK to dim_dates (download day, YYYYMMDD). data_tests: - not_null - relationships: @@ -198,6 +204,62 @@ models: to: ref('dim_dates') field: date_key - name: download_count + description: Downloads recorded for the package that day. + data_tests: + - dbt_utils.expression_is_true: + arguments: + expression: ">= 0" + + - name: fct_contributor_activity_monthly + description: > + Aggregate fact, one row per contributor per month, pivoting the activity event + stream into commit / PR / issue counts. + columns: + - name: activity_key + description: Surrogate key, hash of contributor_key + event_month. + data_tests: [not_null, unique] + - name: contributor_key + description: FK to dim_contributors. + data_tests: + - not_null + - relationships: + arguments: + to: ref('dim_contributors') + field: contributor_key + - name: month_date_key + description: FK to dim_dates (first day of the activity month, YYYYMMDD). + data_tests: + - not_null + - relationships: + arguments: + to: ref('dim_dates') + field: date_key + - name: commits + description: Commits authored that month. + data_tests: + - dbt_utils.expression_is_true: + arguments: + expression: ">= 0" + - name: prs_opened + description: Pull requests opened that month. + data_tests: + - dbt_utils.expression_is_true: + arguments: + expression: ">= 0" + - name: prs_merged + description: Pull requests merged that month. + data_tests: + - dbt_utils.expression_is_true: + arguments: + expression: ">= 0" + - name: issues_opened + description: Issues opened that month. + data_tests: + - dbt_utils.expression_is_true: + arguments: + expression: ">= 0" + - name: issues_closed + description: Issues closed that month. data_tests: - dbt_utils.expression_is_true: arguments: diff --git a/dbt/models/marts/dim_contributors.sql b/dbt/models/marts/dim_contributors.sql index 309f868..aa28800 100644 --- a/dbt/models/marts/dim_contributors.sql +++ b/dbt/models/marts/dim_contributors.sql @@ -1,5 +1,6 @@ -- Type 1 contributor dimension: one row per GitHub login, aggregated across the --- commit and pull-request staging models. Rebuilt in full each run. +-- commit, pull request, and issue staging models. Rebuilt in full each run. +-- Stats are commit/PR-based; issue-only authors appear with zeroed counts. with commits as ( select @@ -22,6 +23,10 @@ contributors as ( select username from commits union select username from prs + union + select author_login as username + from {{ ref('stg_github__issues') }} + where author_login is not null ), commit_stats as ( diff --git a/dbt/models/marts/fct_commits.sql b/dbt/models/marts/fct_commits.sql index a2eac83..f7775a5 100644 --- a/dbt/models/marts/fct_commits.sql +++ b/dbt/models/marts/fct_commits.sql @@ -2,9 +2,23 @@ -- half-open date range (event_date >= valid_from AND < valid_to) so each commit maps -- to the repository version that was current when it landed; to dim_contributors on -- author login; and to dim_dates via an inline YYYYMMDD key. +-- Incremental (delete+insert on commit_key): each run only processes commits at or +-- after the latest committed_at already loaded; the unique key keeps it idempotent. + +{{ + config( + materialized='incremental', + unique_key='commit_key', + incremental_strategy='delete+insert', + on_schema_change='sync_all_columns', + ) +}} with commits as ( select * from {{ ref('stg_github__commits') }} + {% if is_incremental() %} + where committed_at >= (select max(committed_at) from {{ this }}) + {% endif %} ) select diff --git a/dbt/models/marts/fct_contributor_activity_monthly.sql b/dbt/models/marts/fct_contributor_activity_monthly.sql new file mode 100644 index 0000000..ecd1f6e --- /dev/null +++ b/dbt/models/marts/fct_contributor_activity_monthly.sql @@ -0,0 +1,31 @@ +-- Aggregate fact: one row per contributor per month, pivoting the activity event +-- stream into commit / PR / issue counts. Built on int_contributor_activity and the +-- conformed dims (dim_contributors, dim_dates). Foundation for retention analysis. + +with monthly as ( + select + username, + event_month, + count(*) filter (where activity_type = 'commit') as commits, + count(*) filter (where activity_type = 'pr_opened') as prs_opened, + count(*) filter (where activity_type = 'pr_merged') as prs_merged, + count(*) filter (where activity_type = 'issue_opened') as issues_opened, + count(*) filter (where activity_type = 'issue_closed') as issues_closed + from {{ ref('int_contributor_activity') }} + group by username, event_month +) + +select + {{ dbt_utils.generate_surrogate_key(['c.contributor_key', 'm.event_month']) }} + as activity_key, + c.contributor_key, + m.username, + m.event_month, + cast(strftime(m.event_month, '%Y%m%d') as integer) as month_date_key, + m.commits, + m.prs_opened, + m.prs_merged, + m.issues_opened, + m.issues_closed +from monthly m +inner join {{ ref('dim_contributors') }} c on m.username = c.username diff --git a/dbt/models/marts/fct_daily_downloads.sql b/dbt/models/marts/fct_daily_downloads.sql index c5c4090..91d7799 100644 --- a/dbt/models/marts/fct_daily_downloads.sql +++ b/dbt/models/marts/fct_daily_downloads.sql @@ -1,10 +1,26 @@ -- PyPI daily download fact: one row per package per day. -- Filtered to the 'without_mirrors' overall time-series category to avoid double -- counting 'with_mirrors' and to exclude the recent-endpoint 'last_*' aggregates. +-- Incremental (delete+insert on download_key): only processes days at or after the +-- latest download_date already loaded; the unique key keeps it idempotent. + +{{ + config( + materialized='incremental', + unique_key='download_key', + incremental_strategy='delete+insert', + on_schema_change='sync_all_columns', + ) +}} with downloads as ( select * from {{ ref('stg_pypi__downloads') }} where category = 'without_mirrors' + {% if is_incremental() %} + -- date_key is the only date column on the target table; compare the day's key. + and cast(strftime(download_date, '%Y%m%d') as integer) + >= (select max(date_key) from {{ this }}) + {% endif %} ) select diff --git a/dbt/models/staging/github/_github__models.yml b/dbt/models/staging/github/_github__models.yml index 0a1f61e..14cc18a 100644 --- a/dbt/models/staging/github/_github__models.yml +++ b/dbt/models/staging/github/_github__models.yml @@ -26,9 +26,17 @@ models: description: One cleaned row per pull request. columns: - name: pr_number + description: PR number within its repository. data_tests: [not_null] - name: repository + description: Source repository (owner/name), from extraction metadata. data_tests: [not_null] + - name: state + description: PR state. + data_tests: + - accepted_values: + arguments: + values: [open, closed] data_tests: - dbt_utils.unique_combination_of_columns: arguments: @@ -38,9 +46,17 @@ models: description: One cleaned row per issue (pull requests excluded). columns: - name: issue_number + description: Issue number within its repository. data_tests: [not_null] - name: repository + description: Source repository (owner/name), from extraction metadata. data_tests: [not_null] + - name: state + description: Issue state. + data_tests: + - accepted_values: + arguments: + values: [open, closed] data_tests: - dbt_utils.unique_combination_of_columns: arguments: