feat: add intermediate activity stream, monthly aggregate, and incremental facts

Taskfile.yml

@@ -90,3 +90,10 @@ tasks:
     dir: dbt
     cmds:
       - uv run --group dbt dbt build
+
+  dbt:docs:
+    desc: Generate and serve dbt documentation
+    dir: dbt
+    cmds:
+      - uv run --group dbt dbt docs generate
+      - uv run --group dbt dbt docs serve

dbt/dbt_project.yml

@@ -22,6 +22,9 @@ models:
     staging:
       +materialized: view
       +schema: staging
+    intermediate:
+      +materialized: view
+      +schema: intermediate
     marts:
       +materialized: table
       +schema: marts

dbt/models/intermediate/_int__models.yml

@@ -0,0 +1,30 @@
+version: 2
+
+models:
+  - name: int_contributor_activity
+    description: >
+      Tidy contributor activity event stream at username grain - one row per discrete
+      event (commit, pr_opened, pr_merged, issue_opened, issue_closed). Reusable
+      building block for monthly aggregation and PR/health metrics.
+    columns:
+      - name: username
+        description: GitHub login of the actor.
+        data_tests: [not_null]
+      - name: event_date
+        description: Date the activity occurred.
+        data_tests: [not_null]
+      - name: event_month
+        description: First day of the month of event_date.
+        data_tests: [not_null]
+      - name: activity_type
+        description: One of commit, pr_opened, pr_merged, issue_opened, issue_closed.
+        data_tests:
+          - not_null
+          - accepted_values:
+              arguments:
+                values:
+                  - commit
+                  - pr_opened
+                  - pr_merged
+                  - issue_opened
+                  - issue_closed

dbt/models/intermediate/int_contributor_activity.sql

@@ -0,0 +1,68 @@
+-- Tidy contributor activity event stream at username grain: one row per discrete
+-- activity event. A reusable building block for monthly aggregation (and future PR /
+-- health metrics). Each PR and issue contributes up to two events (opened, and
+-- merged/closed) so each lands in the month it actually happened.
+
+with commits as (
+    select
+        author_login as username,
+        committed_at::date as event_date,
+        'commit' as activity_type
+    from {{ ref('stg_github__commits') }}
+    where author_login is not null
+),
+
+prs_opened as (
+    select
+        author_login as username,
+        created_at::date as event_date,
+        'pr_opened' as activity_type
+    from {{ ref('stg_github__pull_requests') }}
+    where author_login is not null
+),
+
+prs_merged as (
+    select
+        author_login as username,
+        merged_at::date as event_date,
+        'pr_merged' as activity_type
+    from {{ ref('stg_github__pull_requests') }}
+    where author_login is not null and merged_at is not null
+),
+
+issues_opened as (
+    select
+        author_login as username,
+        created_at::date as event_date,
+        'issue_opened' as activity_type
+    from {{ ref('stg_github__issues') }}
+    where author_login is not null
+),
+
+issues_closed as (
+    select
+        author_login as username,
+        closed_at::date as event_date,
+        'issue_closed' as activity_type
+    from {{ ref('stg_github__issues') }}
+    where author_login is not null and closed_at is not null
+),
+
+events as (
+    select * from commits
+    union all
+    select * from prs_opened
+    union all
+    select * from prs_merged
+    union all
+    select * from issues_opened
+    union all
+    select * from issues_closed
+)
+
+select
+    username,
+    event_date,
+    date_trunc('month', event_date) as event_month,
+    activity_type
+from events

dbt/models/marts/_marts__models.yml

@@ -185,19 +185,81 @@ models:
                 field: label_key
 
   - name: fct_daily_downloads
-    description: One row per package per day (PyPI 'without_mirrors' downloads).
+    description: >
+      One row per package per day (PyPI 'without_mirrors' downloads). `package` is a
+      degenerate dimension - no package->repository mapping exists yet. Incremental
+      (delete+insert on download_key).
     columns:
       - name: download_key
         description: Surrogate key, hash of package + download_date.
         data_tests: [not_null, unique]
+      - name: package
+        description: PyPI package name (degenerate dimension).
       - name: date_key
+        description: FK to dim_dates (download day, YYYYMMDD).
         data_tests:
           - not_null
           - relationships:
               arguments:
                 to: ref('dim_dates')
                 field: date_key
       - name: download_count
+        description: Downloads recorded for the package that day.
+        data_tests:
+          - dbt_utils.expression_is_true:
+              arguments:
+                expression: ">= 0"
+
+  - name: fct_contributor_activity_monthly
+    description: >
+      Aggregate fact, one row per contributor per month, pivoting the activity event
+      stream into commit / PR / issue counts.
+    columns:
+      - name: activity_key
+        description: Surrogate key, hash of contributor_key + event_month.
+        data_tests: [not_null, unique]
+      - name: contributor_key
+        description: FK to dim_contributors.
+        data_tests:
+          - not_null
+          - relationships:
+              arguments:
+                to: ref('dim_contributors')
+                field: contributor_key
+      - name: month_date_key
+        description: FK to dim_dates (first day of the activity month, YYYYMMDD).
+        data_tests:
+          - not_null
+          - relationships:
+              arguments:
+                to: ref('dim_dates')
+                field: date_key
+      - name: commits
+        description: Commits authored that month.
+        data_tests:
+          - dbt_utils.expression_is_true:
+              arguments:
+                expression: ">= 0"
+      - name: prs_opened
+        description: Pull requests opened that month.
+        data_tests:
+          - dbt_utils.expression_is_true:
+              arguments:
+                expression: ">= 0"
+      - name: prs_merged
+        description: Pull requests merged that month.
+        data_tests:
+          - dbt_utils.expression_is_true:
+              arguments:
+                expression: ">= 0"
+      - name: issues_opened
+        description: Issues opened that month.
+        data_tests:
+          - dbt_utils.expression_is_true:
+              arguments:
+                expression: ">= 0"
+      - name: issues_closed
+        description: Issues closed that month.
         data_tests:
           - dbt_utils.expression_is_true:
               arguments:

dbt/models/marts/dim_contributors.sql

@@ -1,5 +1,6 @@
 -- Type 1 contributor dimension: one row per GitHub login, aggregated across the
--- commit and pull-request staging models. Rebuilt in full each run.
+-- commit, pull request, and issue staging models. Rebuilt in full each run.
+-- Stats are commit/PR-based; issue-only authors appear with zeroed counts.
 
 with commits as (
     select
@@ -22,6 +23,10 @@ contributors as (
     select username from commits
     union
     select username from prs
+    union
+    select author_login as username
+    from {{ ref('stg_github__issues') }}
+    where author_login is not null
 ),
 
 commit_stats as (

dbt/models/marts/fct_commits.sql

@@ -2,9 +2,23 @@
 -- half-open date range (event_date >= valid_from AND < valid_to) so each commit maps
 -- to the repository version that was current when it landed; to dim_contributors on
 -- author login; and to dim_dates via an inline YYYYMMDD key.
+-- Incremental (delete+insert on commit_key): each run only processes commits at or
+-- after the latest committed_at already loaded; the unique key keeps it idempotent.
+
+{{
+    config(
+        materialized='incremental',
+        unique_key='commit_key',
+        incremental_strategy='delete+insert',
+        on_schema_change='sync_all_columns',
+    )
+}}
 
 with commits as (
     select * from {{ ref('stg_github__commits') }}
+    {% if is_incremental() %}
+    where committed_at >= (select max(committed_at) from {{ this }})
+    {% endif %}
 )
 
 select

dbt/models/marts/fct_contributor_activity_monthly.sql

@@ -0,0 +1,31 @@
+-- Aggregate fact: one row per contributor per month, pivoting the activity event
+-- stream into commit / PR / issue counts. Built on int_contributor_activity and the
+-- conformed dims (dim_contributors, dim_dates). Foundation for retention analysis.
+
+with monthly as (
+    select
+        username,
+        event_month,
+        count(*) filter (where activity_type = 'commit') as commits,
+        count(*) filter (where activity_type = 'pr_opened') as prs_opened,
+        count(*) filter (where activity_type = 'pr_merged') as prs_merged,
+        count(*) filter (where activity_type = 'issue_opened') as issues_opened,
+        count(*) filter (where activity_type = 'issue_closed') as issues_closed
+    from {{ ref('int_contributor_activity') }}
+    group by username, event_month
+)
+
+select
+    {{ dbt_utils.generate_surrogate_key(['c.contributor_key', 'm.event_month']) }}
+        as activity_key,
+    c.contributor_key,
+    m.username,
+    m.event_month,
+    cast(strftime(m.event_month, '%Y%m%d') as integer) as month_date_key,
+    m.commits,
+    m.prs_opened,
+    m.prs_merged,
+    m.issues_opened,
+    m.issues_closed
+from monthly m
+inner join {{ ref('dim_contributors') }} c on m.username = c.username

dbt/models/marts/fct_daily_downloads.sql

@@ -1,10 +1,26 @@
 -- PyPI daily download fact: one row per package per day.
 -- Filtered to the 'without_mirrors' overall time-series category to avoid double
 -- counting 'with_mirrors' and to exclude the recent-endpoint 'last_*' aggregates.
+-- Incremental (delete+insert on download_key): only processes days at or after the
+-- latest download_date already loaded; the unique key keeps it idempotent.
+
+{{
+    config(
+        materialized='incremental',
+        unique_key='download_key',
+        incremental_strategy='delete+insert',
+        on_schema_change='sync_all_columns',
+    )
+}}
 
 with downloads as (
     select * from {{ ref('stg_pypi__downloads') }}
     where category = 'without_mirrors'
+    {% if is_incremental() %}
+    -- date_key is the only date column on the target table; compare the day's key.
+    and cast(strftime(download_date, '%Y%m%d') as integer)
+        >= (select max(date_key) from {{ this }})
+    {% endif %}
 )
 
 select

dbt/models/staging/github/_github__models.yml

@@ -26,9 +26,17 @@ models:
     description: One cleaned row per pull request.
     columns:
       - name: pr_number
+        description: PR number within its repository.
         data_tests: [not_null]
       - name: repository
+        description: Source repository (owner/name), from extraction metadata.
         data_tests: [not_null]
+      - name: state
+        description: PR state.
+        data_tests:
+          - accepted_values:
+              arguments:
+                values: [open, closed]
     data_tests:
       - dbt_utils.unique_combination_of_columns:
           arguments:
@@ -38,9 +46,17 @@ models:
     description: One cleaned row per issue (pull requests excluded).
     columns:
       - name: issue_number
+        description: Issue number within its repository.
         data_tests: [not_null]
       - name: repository
+        description: Source repository (owner/name), from extraction metadata.
         data_tests: [not_null]
+      - name: state
+        description: Issue state.
+        data_tests:
+          - accepted_values:
+              arguments:
+                values: [open, closed]
     data_tests:
       - dbt_utils.unique_combination_of_columns:
           arguments: