diff --git a/dbt/models/marts/_marts__models.yml b/dbt/models/marts/_marts__models.yml index d538b04..47b9a8d 100644 --- a/dbt/models/marts/_marts__models.yml +++ b/dbt/models/marts/_marts__models.yml @@ -147,6 +147,31 @@ models: arguments: expression: ">= 0" + - name: fct_releases + description: > + One row per published release. Repository resolved via the SCD2 half-open range + on the publish date. Incremental (delete+insert on release_key). An event fact + analyzed by counting (release cadence), with no additive measure. + columns: + - name: release_key + description: Surrogate key, hash of repository + release_id. + data_tests: [not_null, unique] + - name: repository_key + data_tests: + - not_null + - relationships: + arguments: + to: ref('dim_repositories') + field: repository_key + - name: date_key + description: FK to dim_dates (publish day, YYYYMMDD). + data_tests: + - not_null + - relationships: + arguments: + to: ref('dim_dates') + field: date_key + - name: bridge_issue_labels description: Bridge for the issue<->label many-to-many. Grain - one row per issue/label. data_tests: @@ -203,6 +228,50 @@ models: arguments: expression: ">= 0" + - name: fct_repository_metrics + description: > + Periodic-snapshot fact: one row per repository per capture (ingestion) day, + recording GitHub stars/forks/open_issues. Accumulated incrementally + (delete+insert on metric_key). + columns: + - name: metric_key + description: Surrogate key, hash of repository_id + capture date. + data_tests: [not_null, unique] + - name: repository_key + description: FK to dim_repositories (version current on the capture date). + data_tests: + - not_null + - relationships: + arguments: + to: ref('dim_repositories') + field: repository_key + - name: date_key + description: FK to dim_dates (capture day, YYYYMMDD). + data_tests: + - not_null + - relationships: + arguments: + to: ref('dim_dates') + field: date_key + - name: stars + description: Stargazer count at capture time. + data_tests: + - dbt_utils.expression_is_true: + arguments: + expression: ">= 0" + - name: forks + description: Fork count at capture time. + data_tests: + - dbt_utils.expression_is_true: + arguments: + expression: ">= 0" + - name: open_issues + description: Open issue (+PR) count at capture time. + data_tests: + - dbt_utils.expression_is_true: + arguments: + expression: ">= 0" + - name: fct_contributor_activity_monthly description: > Aggregate fact, one row per contributor per month, pivoting the activity event diff --git a/dbt/models/marts/dim_repositories.sql b/dbt/models/marts/dim_repositories.sql index 57f3a96..3a8891b 100644 --- a/dbt/models/marts/dim_repositories.sql +++ b/dbt/models/marts/dim_repositories.sql @@ -9,9 +9,6 @@ with versioned as ( name, owner_login, description, - stars, - forks, - open_issues, language, license_spdx, topics, @@ -43,9 +40,6 @@ select name, owner_login, description, - stars, - forks, - open_issues, language, license_spdx, topics, diff --git a/dbt/models/marts/fct_releases.sql b/dbt/models/marts/fct_releases.sql new file mode 100644 index 0000000..aedcfad --- /dev/null +++ b/dbt/models/marts/fct_releases.sql @@ -0,0 +1,37 @@ +-- Release activity fact: one row per published release. Joins to dim_repositories via +-- the SCD2 half-open date range (event_date >= valid_from AND < valid_to) so each +-- release maps to the repository version current when it was published; to dim_dates +-- via an inline YYYYMMDD key on the publish date. An event fact with no additive +-- measure - analyze by counting (release cadence, time between releases). +-- Incremental (delete+insert on release_key) keyed on the monotonic dlt `_loaded_at` +-- (publish dates can be backdated); the unique key keeps it idempotent. + +{{ + config( + materialized='incremental', + unique_key='release_key', + incremental_strategy='delete+insert', + on_schema_change='sync_all_columns', + ) +}} + +with releases as ( + select * from {{ ref('stg_github__releases') }} + -- Drafts have no publish date; exclude them so date_key / repository_key resolve. + where published_at is not null + {% if is_incremental() %} + and _loaded_at >= (select max(_loaded_at) from {{ this }}) + {% endif %} +) + +select + {{ dbt_utils.generate_surrogate_key(['rel.repository', 'rel.release_id']) }} + as release_key, + r.repository_key, + {{ date_key('rel.published_at') }} as date_key, + rel.tag_name, + rel.name as release_name, + rel.published_at, + rel._loaded_at +from releases rel +{{ scd2_repository_join('rel.repository', 'rel.published_at::date') }} diff --git a/dbt/models/marts/fct_repository_metrics.sql b/dbt/models/marts/fct_repository_metrics.sql new file mode 100644 index 0000000..72bb818 --- /dev/null +++ b/dbt/models/marts/fct_repository_metrics.sql @@ -0,0 +1,40 @@ +-- GitHub repository metrics fact: a periodic-snapshot fact at one row per repository +-- per capture (ingestion) day, recording the volatile counters stars/forks/open_issues. +-- GitHub's API returns only the *current* counter values (no history), so history is +-- accumulated incrementally - each daily run appends that day's values, dated by the +-- dlt load timestamp. `repository_key` resolves through the SCD2 dim_repositories +-- half-open range for the capture date. Incremental (delete+insert on metric_key) +-- keeps same-day re-runs idempotent. + +{{ + config( + materialized='incremental', + unique_key='metric_key', + incremental_strategy='delete+insert', + on_schema_change='sync_all_columns', + ) +}} + +with repositories as ( + select + *, + cast(_loaded_at as date) as capture_date + from {{ ref('stg_github__repositories') }} + {% if is_incremental() %} + -- Only (re)load capture days at or after the latest already loaded. + where {{ date_key('_loaded_at') }} >= (select max(date_key) from {{ this }}) + {% endif %} +) + +select + {{ dbt_utils.generate_surrogate_key(['m.repository_id', 'm.capture_date']) }} + as metric_key, + m.repository_id, + m.repository_name, + r.repository_key, + {{ date_key('m.capture_date') }} as date_key, + m.stars, + m.forks, + m.open_issues +from repositories m +{{ scd2_repository_join('m.repository_name', 'm.capture_date') }} diff --git a/dbt/snapshots/snap_repositories.sql b/dbt/snapshots/snap_repositories.sql index b8b6f8f..633182d 100644 --- a/dbt/snapshots/snap_repositories.sql +++ b/dbt/snapshots/snap_repositories.sql @@ -4,7 +4,7 @@ config( unique_key='repository_id', strategy='check', - check_cols=['stars', 'forks', 'open_issues', 'description', 'topics'], + check_cols=['description', 'language', 'license_spdx', 'topics'], ) }}