diff --git a/dags/repolytics_daily.py b/dags/repolytics_daily.py index 930c91e..5790f48 100644 --- a/dags/repolytics_daily.py +++ b/dags/repolytics_daily.py @@ -25,7 +25,6 @@ logger = logging.getLogger(__name__) -# Location of the dbt project inside the container DBT_PROJECT_DIR = Path(os.environ.get("DBT_PROJECT_DIR", "/opt/airflow/dbt")) @@ -65,11 +64,10 @@ def log_task_failure(context) -> None: def repolytics_daily(): @task(multiple_outputs=False) def ingest_github() -> dict[str, int]: - """Run GitHub incremental ingestion into the DuckDB `raw` dataset. + """Run GitHub incremental ingestion, returning per-table row counts. - Skipped on historical backfill runs: the dlt cursor is global (not - per-interval), so replaying old intervals for GitHub is meaningless. - Returns per-table row counts (pushed to XCom for the summary task). + Skipped on historical backfill runs: the dlt cursor is global, so replaying + old intervals is meaningless. """ from airflow.sdk import get_current_context @@ -86,12 +84,7 @@ def ingest_github() -> dict[str, int]: @task(multiple_outputs=False) def ingest_pypi() -> dict[str, int]: - """Run PyPI ingestion for the run's data-interval day (BigQuery). - - Uses `data_interval_start`, so each run loads exactly one day and - the DAG backfills cleanly per interval. Returns per-table row counts - (pushed to XCom for the summary task). - """ + """Run PyPI ingestion for the run's `data_interval_start` day.""" from airflow.sdk import get_current_context from repolytics.ingestion.pipeline import run_pypi diff --git a/dashboard/lib/data.py b/dashboard/lib/data.py index e73092a..d98322f 100644 --- a/dashboard/lib/data.py +++ b/dashboard/lib/data.py @@ -10,10 +10,7 @@ def connect(duckdb_path: str | Path | None = None) -> duckdb.DuckDBPyConnection: - """Open a read-only DuckDB connection to the warehouse. - - Defaults to `Settings.duckdb_path`. - """ + """Open a read-only DuckDB connection to the warehouse.""" path = Path(duckdb_path) if duckdb_path is not None else get_settings().duckdb_path return duckdb.connect(str(path), read_only=True) @@ -35,11 +32,8 @@ def has_marts(conn: duckdb.DuckDBPyConnection) -> bool: def date_bounds(conn: duckdb.DuckDBPyConnection) -> tuple[date, date]: - """Min/max event date across all activity facts, for the sidebar date filter. - - Spans commits, PRs (opened), issues (opened), and releases so the slider covers - every date-filtered chart - a repo whose PRs/releases extend past its last commit - is not clipped. + """Min/max event date across all activity facts (commits, PRs, issues, releases), + for the sidebar date filter, so no date-filtered chart is clipped. """ row = conn.execute( """ diff --git a/dashboard/lib/health.py b/dashboard/lib/health.py index 6cef552..423e640 100644 --- a/dashboard/lib/health.py +++ b/dashboard/lib/health.py @@ -1,17 +1,12 @@ """Composite project-health score (streamlit-free, pure DataFrame -> DataFrame). -A blend of five signals, each scored **absolutely** against a fixed reference target. -That keeps a project's score stable regardless of which other projects are selected, -and comparable across sessions and over time. The two rate signals (PR merge rate, -issue close rate) are already 0..1 and are used directly; the unbounded signals -(recent commits, stars) are mapped through a saturating log transform against a -reference, and release recency through an exponential time-decay. - -Input columns (one row per repo, from `queries.health_components`): - repository_name, commits_90d, pr_merge_rate, issue_close_rate, - days_since_release (nullable -> treated as never released -> 0), stars - -Output adds the five `c_*` component scores (0..1) and `health_score` (0..100). +A weighted blend of five signals, each scored **absolutely** against a fixed +reference so a project's score is stable regardless of the current selection. Rate +signals (PR merge/issue close) are used directly; unbounded signals (commits, stars) +go through a saturating log transform and release recency through exponential decay. + +Input columns come from `queries.health_components`; output adds the five `c_*` +component scores (0..1) and `health_score` (0..100). """ import numpy as np diff --git a/dashboard/lib/queries.py b/dashboard/lib/queries.py index 32f5d31..5a0d42c 100644 --- a/dashboard/lib/queries.py +++ b/dashboard/lib/queries.py @@ -368,9 +368,8 @@ def contributor_leaderboard( def retention_cohort(conn: duckdb.DuckDBPyConnection) -> pd.DataFrame: """Monthly contributor retention: % of a first-active cohort active N months on. - Necessarily **repo-agnostic**: that mart's grain is contributor x month across all - tracked projects (no repository_key), so retention spans the whole tracked set and - cannot be filtered by repo. First activity across the set sets each cohort. + Repo-agnostic: the mart's grain is contributor x month with no repository_key, so + retention spans the whole tracked set and cannot be filtered by repo. """ sql = """ with active as ( @@ -404,8 +403,7 @@ def retention_cohort(conn: duckdb.DuckDBPyConnection) -> pd.DataFrame: def active_contributors_monthly(conn: duckdb.DuckDBPyConnection) -> pd.DataFrame: """Monthly active contributors, split into new vs. returning. - Repo-agnostic for the same reason as `retention_cohort` (the mart has no - repository grain). + Repo-agnostic like `retention_cohort` (the mart has no repository grain). """ sql = """ with monthly as ( diff --git a/dbt/macros/scd2_repository_join.sql b/dbt/macros/scd2_repository_join.sql index 574d506..988dbb8 100644 --- a/dbt/macros/scd2_repository_join.sql +++ b/dbt/macros/scd2_repository_join.sql @@ -1,10 +1,7 @@ {#- - Resolve a fact repository_key against the SCD2 dim_repositories using a - half-open date range (event_date >= valid_from AND < valid_to), so each event - maps to the repository version that was current when it occurred. Centralizes - the SCD2 join contract (half-open bounds; earliest version backdated, current - version valid_to = 9999-12-31) that every fact shares. Select - `.repository_key` after using this. + Resolve a fact repository_key against SCD2 dim_repositories using a half-open date + range (event_date >= valid_from AND < valid_to), so each event maps to the + repository version current when it occurred. Select `.repository_key` after. repo_expr - SQL expression for the repository full name (owner/name) date_expr - SQL expression for the event date (cast to date) diff --git a/dbt/macros/source_or_empty.sql b/dbt/macros/source_or_empty.sql index ec0ab09..f2fc8f1 100644 --- a/dbt/macros/source_or_empty.sql +++ b/dbt/macros/source_or_empty.sql @@ -1,13 +1,10 @@ {#- - Return `select * from ` when the source relation exists, otherwise an - empty result with the given columns (`select cast(null as ) as ... where false`). + Return `select * from ` when the source relation exists, otherwise a typed + empty result. dlt only materializes a child table when some row populated the list, + so a list empty across every ingested row leaves the child table absent; this keeps + the build resilient to that case. - dlt only materializes a normalized child table when some row populated the - underlying list, so a list that is empty across every ingested row leaves the - child table absent. Wrapping the source in this macro keeps the build resilient - to that case. - - `columns` is a mapping of column name -> SQL type, covering the columns the caller reads. + `columns` is a mapping of column name -> SQL type covering the columns the caller reads. -#} {% macro source_or_empty(source_name, table_name, columns) -%} {%- set rel = source(source_name, table_name) -%} diff --git a/dbt/models/intermediate/int_contributor_activity.sql b/dbt/models/intermediate/int_contributor_activity.sql index cb988d0..2593146 100644 --- a/dbt/models/intermediate/int_contributor_activity.sql +++ b/dbt/models/intermediate/int_contributor_activity.sql @@ -1,7 +1,6 @@ --- Tidy contributor activity event stream at username grain: one row per discrete --- activity event. A reusable building block for monthly aggregation (and future PR / --- health metrics). Each PR and issue contributes up to two events (opened, and --- merged/closed) so each lands in the month it actually happened. +-- Contributor activity event stream at username grain: one row per activity event, +-- reused for monthly aggregation. Each PR/issue contributes up to two events (opened +-- and merged/closed) so each lands in the month it happened. with commits as ( select diff --git a/dbt/models/marts/_marts__models.yml b/dbt/models/marts/_marts__models.yml index d9733f6..a371302 100644 --- a/dbt/models/marts/_marts__models.yml +++ b/dbt/models/marts/_marts__models.yml @@ -533,17 +533,17 @@ models: to: ref('dim_dates') field: date_key - name: commits - description: Commits authored that month (count, non-negative by construction). + description: Commits authored that month. data_type: bigint - name: prs_opened - description: Pull requests opened that month (count, non-negative by construction). + description: Pull requests opened that month. data_type: bigint - name: prs_merged - description: Pull requests merged that month (count, non-negative by construction). + description: Pull requests merged that month. data_type: bigint - name: issues_opened - description: Issues opened that month (count, non-negative by construction). + description: Issues opened that month. data_type: bigint - name: issues_closed - description: Issues closed that month (count, non-negative by construction). + description: Issues closed that month. data_type: bigint diff --git a/dbt/models/marts/bridge_issue_labels.sql b/dbt/models/marts/bridge_issue_labels.sql index d4e65b0..fbfe047 100644 --- a/dbt/models/marts/bridge_issue_labels.sql +++ b/dbt/models/marts/bridge_issue_labels.sql @@ -1,7 +1,5 @@ --- Bridge table for the issue<->label many-to-many. Built from the flattened issue --- label staging model; recomputes the issue surrogate key the same way as fct_issues --- and resolves each label name to its label_key in dim_labels. --- Grain: one row per (issue, label). +-- Bridge for the issue<->label many-to-many; one row per (issue, label). +-- Recomputes the issue surrogate key the same way as fct_issues. with issue_labels as ( select * from {{ ref('stg_github__issue_labels') }} diff --git a/dbt/models/marts/dim_contributors.sql b/dbt/models/marts/dim_contributors.sql index aa28800..a133c5d 100644 --- a/dbt/models/marts/dim_contributors.sql +++ b/dbt/models/marts/dim_contributors.sql @@ -1,5 +1,4 @@ --- Type 1 contributor dimension: one row per GitHub login, aggregated across the --- commit, pull request, and issue staging models. Rebuilt in full each run. +-- Type 1 contributor dimension: one row per GitHub login, rebuilt each run. -- Stats are commit/PR-based; issue-only authors appear with zeroed counts. with commits as ( diff --git a/dbt/models/marts/dim_repositories.sql b/dbt/models/marts/dim_repositories.sql index 3a8891b..60d523c 100644 --- a/dbt/models/marts/dim_repositories.sql +++ b/dbt/models/marts/dim_repositories.sql @@ -14,20 +14,17 @@ with versioned as ( topics, created_at, updated_at, - -- Full-resolution version timestamp drives the surrogate key so it stays - -- unique even if a repo changes more than once on the same calendar day; - -- valid_from/valid_to are kept at date grain for fact date-range joins. + -- Full-resolution timestamp keys the surrogate so it stays unique across + -- same-day changes; valid_from/valid_to are date grain for fact joins. dbt_valid_from as version_ts, - -- The earliest version opens at a past-infinity sentinel ('1900-01-01') - -- so historical facts that predate the first snapshot still resolve to - -- the first captured version instead of dropping. + -- Earliest version opens at a past-infinity sentinel so facts predating the + -- first snapshot still resolve to it instead of dropping. case when row_number() over ( partition by repository_id order by dbt_valid_from ) = 1 then date '1900-01-01' else dbt_valid_from::date end as valid_from, - -- '9999-12-31' = future-infinity bound coalesce(dbt_valid_to::date, date '9999-12-31') as valid_to, dbt_valid_to is null as is_current from {{ ref('snap_repositories') }} diff --git a/dbt/models/marts/fct_commits.sql b/dbt/models/marts/fct_commits.sql index 596f57c..d2cb982 100644 --- a/dbt/models/marts/fct_commits.sql +++ b/dbt/models/marts/fct_commits.sql @@ -1,11 +1,5 @@ --- Commit activity fact: one row per commit. Joins to dim_repositories via the SCD2 --- half-open date range (event_date >= valid_from AND < valid_to) so each commit maps --- to the repository version that was current when it landed; to dim_contributors on --- author login; and to dim_dates via an inline YYYYMMDD key. --- Incremental (delete+insert on commit_key): each run only processes rows loaded --- since the last run, keyed on the monotonic dlt `_loaded_at` (the git author date is --- not monotonic, so a rebased/old-authored commit pushed today would be skipped). --- The unique key keeps it idempotent. +-- Commit activity fact: one row per commit, resolving repository via the SCD2 join. +-- Incremental on the dlt `_loaded_at` because the git author date isn't monotonic. {{ config( diff --git a/dbt/models/marts/fct_contributor_activity_monthly.sql b/dbt/models/marts/fct_contributor_activity_monthly.sql index 0196e08..c233e78 100644 --- a/dbt/models/marts/fct_contributor_activity_monthly.sql +++ b/dbt/models/marts/fct_contributor_activity_monthly.sql @@ -1,6 +1,5 @@ --- Aggregate fact: one row per contributor per month, pivoting the activity event --- stream into commit / PR / issue counts. Built on int_contributor_activity and the --- conformed dims (dim_contributors, dim_dates). Foundation for retention analysis. +-- Aggregate fact: one row per contributor per month, pivoting int_contributor_activity +-- into commit / PR / issue counts. Foundation for retention analysis. with monthly as ( select diff --git a/dbt/models/marts/fct_daily_downloads.sql b/dbt/models/marts/fct_daily_downloads.sql index d67cde0..34ed2ba 100644 --- a/dbt/models/marts/fct_daily_downloads.sql +++ b/dbt/models/marts/fct_daily_downloads.sql @@ -1,8 +1,6 @@ --- PyPI daily download fact: one row per package per day. `repository_key` is resolved --- through the `projects` seed (package -> repo) and the SCD2 dim_repositories half-open --- range; it is nullable for packages with no mapped/ingested repo. Incremental --- (delete+insert on download_key): only processes days at or after the latest --- download_date already loaded; the unique key keeps it idempotent. +-- PyPI daily download fact: one row per package per day. `repository_key` resolves +-- through the `projects` seed (package -> repo) and the SCD2 join; nullable for +-- unmapped packages. Incremental (delete+insert on download_key) keeps it idempotent. {{ config( @@ -16,7 +14,6 @@ with downloads as ( select * from {{ ref('stg_pypi__downloads') }} {% if is_incremental() %} - -- date_key is the only date column on the target table; compare the day's key. where {{ date_key('download_date') }} >= (select max(date_key) from {{ this }}) {% endif %} ) diff --git a/dbt/models/marts/fct_issues.sql b/dbt/models/marts/fct_issues.sql index daebf34..6e9ad6a 100644 --- a/dbt/models/marts/fct_issues.sql +++ b/dbt/models/marts/fct_issues.sql @@ -1,6 +1,5 @@ --- Issue lifecycle fact: one row per issue. Repository resolved via the SCD2 half-open --- range on the issue open date; author via dim_contributors; opened/closed date keys --- are inline YYYYMMDD references to dim_dates (closed is nullable). +-- Issue lifecycle fact: one row per issue, resolving repository via the SCD2 join on +-- the open date (closed_date_key is nullable for open issues). with issues as ( select * from {{ ref('stg_github__issues') }} diff --git a/dbt/models/marts/fct_pull_requests.sql b/dbt/models/marts/fct_pull_requests.sql index 88b75e8..5722c10 100644 --- a/dbt/models/marts/fct_pull_requests.sql +++ b/dbt/models/marts/fct_pull_requests.sql @@ -1,6 +1,5 @@ --- Pull request lifecycle fact: one row per PR. Repository resolved via the SCD2 --- half-open range on the PR open date; author via dim_contributors; opened/merged --- date keys are inline YYYYMMDD references to dim_dates (merged is nullable). +-- Pull request lifecycle fact: one row per PR, resolving repository via the SCD2 join +-- on the open date (merged_date_key is nullable for unmerged PRs). with pull_requests as ( select * from {{ ref('stg_github__pull_requests') }} diff --git a/dbt/models/marts/fct_releases.sql b/dbt/models/marts/fct_releases.sql index 0efda4e..d6cb159 100644 --- a/dbt/models/marts/fct_releases.sql +++ b/dbt/models/marts/fct_releases.sql @@ -1,10 +1,6 @@ --- Release activity fact: one row per published release. Joins to dim_repositories via --- the SCD2 half-open date range (event_date >= valid_from AND < valid_to) so each --- release maps to the repository version current when it was published; to dim_dates --- via an inline YYYYMMDD key on the publish date. An event fact with no additive --- measure - analyze by counting (release cadence, time between releases). --- Incremental (delete+insert on release_key) keyed on the monotonic dlt `_loaded_at` --- (publish dates can be backdated); the unique key keeps it idempotent. +-- Release activity fact: one row per published release, resolving repository via the +-- SCD2 join on the publish date. No additive measure - analyze by counting (cadence). +-- Incremental on the monotonic dlt `_loaded_at` (publish dates can be backdated). {{ config( diff --git a/dbt/models/marts/fct_repository_metrics.sql b/dbt/models/marts/fct_repository_metrics.sql index ca18388..f027752 100644 --- a/dbt/models/marts/fct_repository_metrics.sql +++ b/dbt/models/marts/fct_repository_metrics.sql @@ -1,10 +1,7 @@ --- GitHub repository metrics fact: a periodic-snapshot fact at one row per repository --- per capture (ingestion) day, recording the volatile counters stars/forks/open_issues. --- GitHub's API returns only the *current* counter values (no history), so history is --- accumulated incrementally - each daily run appends that day's values, dated by the --- dlt load timestamp. `repository_key` resolves through the SCD2 dim_repositories --- half-open range for the capture date. Incremental (delete+insert on metric_key) --- keeps same-day re-runs idempotent. +-- Periodic-snapshot fact: one row per repository per capture day, recording the +-- volatile stars/forks/open_issues. GitHub returns only current values (no history), +-- so each daily run appends that day's snapshot. Incremental (delete+insert on +-- metric_key) keeps same-day re-runs idempotent. {{ config( diff --git a/dbt/models/staging/github/stg_github__commits.sql b/dbt/models/staging/github/stg_github__commits.sql index 45557c3..15ac939 100644 --- a/dbt/models/staging/github/stg_github__commits.sql +++ b/dbt/models/staging/github/stg_github__commits.sql @@ -26,5 +26,4 @@ select c._loaded_at from commits c left join parent_counts p on c._dlt_id = p._dlt_parent_id --- Keep non-merge commits (<= 1 parent); commits with no parents are kept too. where coalesce(p.parent_count, 0) <= 1 diff --git a/dbt/models/staging/github/stg_github__issues.sql b/dbt/models/staging/github/stg_github__issues.sql index 4b25f2f..7740fe4 100644 --- a/dbt/models/staging/github/stg_github__issues.sql +++ b/dbt/models/staging/github/stg_github__issues.sql @@ -1,6 +1,5 @@ --- Structural cleaning over dlt's normalized `issues` table. The issues endpoint --- returns PRs too; drop them (real issues have no `pull_request`). Labels live in --- the `issues__labels` child table (see stg_github__issue_labels). +-- Structural cleaning over dlt's normalized `issues` table. The endpoint returns PRs +-- too; drop them (real issues have no `pull_request`). Labels: see stg_github__issue_labels. with issues as ( select * from {{ source('github', 'issues') }} diff --git a/dbt/models/staging/github/stg_github__repositories.sql b/dbt/models/staging/github/stg_github__repositories.sql index 656769a..b05e0dc 100644 --- a/dbt/models/staging/github/stg_github__repositories.sql +++ b/dbt/models/staging/github/stg_github__repositories.sql @@ -1,6 +1,6 @@ -- Structural cleaning over dlt's normalized `repositories` table. Topics are --- re-aggregated from the dlt child table into a sorted comma string so the SCD2 --- snapshot's check_cols see a stable scalar. +-- re-aggregated from the child table into a sorted comma string so the snapshot's +-- check_cols see a stable scalar. with repositories as ( select * from {{ source('github', 'repositories') }} diff --git a/pyproject.toml b/pyproject.toml index 942fa9d..ac7e90b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,8 +41,8 @@ line-length = 88 select = ["E", "W", "F", "I", "UP", "B", "C4", "SIM", "PTH", "RUF"] [tool.ruff.lint.isort] -# `lib`/`views` are the dashboard app's own packages (imported relative to the -# dashboard/ dir, which Streamlit puts on sys.path), so group them as first-party. +# `lib`/`views` are the dashboard app's own packages (relative to dashboard/ on +# Streamlit's sys.path), so treat them as first-party. known-first-party = ["lib", "views"] [tool.pytest.ini_options] diff --git a/src/repolytics/config.py b/src/repolytics/config.py index 37f6a0a..4f70c3d 100644 --- a/src/repolytics/config.py +++ b/src/repolytics/config.py @@ -34,10 +34,8 @@ class Settings(BaseSettings): @cached_property def projects(self) -> list[dict[str, str]]: - """Projects to ingest, each a `{repo, package}` row from `projects_file`. - - `package` may be blank for repos not published to PyPI. Returns an empty - list when the file is absent. + """Projects to ingest as `{repo, package}` rows (empty if the file is absent; + `package` may be blank for repos not on PyPI). """ if not self.projects_file.exists(): return [] diff --git a/src/repolytics/ingestion/_meta.py b/src/repolytics/ingestion/_meta.py index 1f5aca7..e744a4b 100644 --- a/src/repolytics/ingestion/_meta.py +++ b/src/repolytics/ingestion/_meta.py @@ -1,9 +1,8 @@ """Shared extraction-metadata stamping for dlt resources. -dlt does not attribute sub-resource rows (commits/issues to their repo) or add a -per-row load timestamp by default, so each resource maps its records through -`stamp(...)` to inject `_loaded_at` plus constant `_`-prefixed provenance columns -(`_repo` for GitHub sub-resources, `_package` for PyPI). +dlt does not attribute sub-resource rows to their repo or add a load timestamp, so +resources map records through `stamp(...)` to inject `_loaded_at` plus constant +`_`-prefixed provenance columns (`_repo`, `_package`). """ from collections.abc import Callable @@ -11,10 +10,8 @@ def stamp(**metadata: str) -> Callable[[dict], dict]: - """Build a dlt `add_map` function that stamps `_loaded_at` + `metadata`. - - Returns a callable applied to each record; it adds a UTC `_loaded_at` and one - constant column per `metadata` key (e.g. `_repo`/`_package`). + """Build a dlt `add_map` callable that adds a UTC `_loaded_at` and one constant + column per `metadata` key (e.g. `_repo`/`_package`) to each record. """ def _apply(record: dict) -> dict: diff --git a/src/repolytics/ingestion/github_source.py b/src/repolytics/ingestion/github_source.py index 5320cf6..7bc0994 100644 --- a/src/repolytics/ingestion/github_source.py +++ b/src/repolytics/ingestion/github_source.py @@ -49,10 +49,8 @@ def commits( "commit.author.date" ), ) -> Iterator[dict]: - # `since` (server-side, on commit date) bounds the fetch; dlt filters + merge - # keep it idempotent. The cursor is global across repos and commit dates can be - # backdated (rebases), so a backdated commit pushed today may be missed - the - # same trade-off the fct_commits watermark documents. + # `since` bounds the fetch server-side on commit date. The cursor is global + # across repos, so a backdated commit (rebase) pushed today can be missed. params = {"since": updated.last_value} if updated.last_value else {} for repo in repos: owner, name = repo.split("/") @@ -62,8 +60,8 @@ def commits( name="issues", write_disposition="merge", primary_key=["_repo", "number"], # issue number is unique per repo - # Force the flattened PR marker column to exist even when a load has no - # PR-shaped issues, so the staging PR filter never references a missing column. + # Force the PR marker column to exist so the staging PR filter never + # references a missing column when a load has no PR-shaped issues. columns={"pull_request__url": {"data_type": "text", "nullable": True}}, ) def issues( @@ -90,10 +88,8 @@ def pull_requests( "updated_at" ), ) -> Iterator[dict]: - # The /pulls endpoint has no `since`, so we can't bound the fetch server-side - # and can't use dlt's row_order early-exit either (it assumes one monotonic - # stream, but looping repos makes the updated_at sequence saw-tooth). We page - # the PR list and let dlt's cursor + merge drop/upsert unchanged rows. + # The /pulls endpoint has no `since`; instead we page everything and + # let dlt's cursor + merge drop/upsert unchanged rows. params = {"state": "all", "sort": "updated", "direction": "desc"} for repo in repos: owner, name = repo.split("/") diff --git a/src/repolytics/ingestion/pipeline.py b/src/repolytics/ingestion/pipeline.py index e751c95..29fca36 100644 --- a/src/repolytics/ingestion/pipeline.py +++ b/src/repolytics/ingestion/pipeline.py @@ -22,22 +22,13 @@ def build_pipeline(settings: Settings) -> dlt.Pipeline: def _table_row_counts(pipeline: dlt.Pipeline) -> dict[str, int]: - """Rows loaded per table in the last run, excluding dlt's internal tables. - - Reads `last_trace.last_normalize_info.row_counts` (table name -> count) and - drops the `_dlt_*` bookkeeping tables so the summary reflects only ingested data. - """ + """Rows loaded per table in the last run, excluding dlt's `_dlt_*` tables.""" counts = pipeline.last_trace.last_normalize_info.row_counts return {table: n for table, n in counts.items() if not table.startswith("_dlt")} def run_github(settings: Settings | None = None) -> dict[str, int]: - """Run GitHub ingestion into the configured DuckDB warehouse. - - No-ops (logs and returns `{}`) when no repos are configured, so an empty repo - list succeeds as a skip rather than failing. Returns the per-table row counts - loaded in this run. - """ + """Run GitHub ingestion, returning per-table row counts (`{}` if no repos).""" settings = settings or get_settings() repos = settings.target_repos @@ -55,11 +46,9 @@ def run_github(settings: Settings | None = None) -> dict[str, int]: def run_pypi( settings: Settings | None = None, target_date: date | None = None ) -> dict[str, int]: - """Run PyPI ingestion for a single day into the configured DuckDB warehouse. - - Queries the BigQuery public dataset for `target_date` (defaults to yesterday - UTC, the most recent complete partition). No-ops (logs and returns `{}`) when - no packages are configured. Returns the per-table row counts loaded in this run. + """Run PyPI ingestion for `target_date` (defaults to yesterday UTC, the most + recent complete partition), returning per-table row counts (no-ops to `{}` if no + packages). """ settings = settings or get_settings() diff --git a/src/repolytics/ingestion/pypi_source.py b/src/repolytics/ingestion/pypi_source.py index e2fe7f8..8fb763d 100644 --- a/src/repolytics/ingestion/pypi_source.py +++ b/src/repolytics/ingestion/pypi_source.py @@ -15,13 +15,12 @@ TABLE = "bigquery-public-data.pypi.file_downloads" -# Bulk-mirror installers excluded so counts reflect real installs, matching pypistats' -# "without_mirrors" definition. +# Bulk-mirror installers excluded to match pypistats' "without_mirrors" definition. # See: https://pypistats.org/faqs#what-is-the-difference-between-without_mirrors-and-with_mirrors MIRROR_INSTALLERS = ["bandersnatch", "z3c.pypimirror", "Artifactory", "devpi"] -# Hard ceiling on bytes scanned per query. A partition-pruned single-day query stays -# well under this, so hitting it means the date filter was lost +# Cost guard: a pruned single-day query stays well under this, so hitting it means +# the date filter was lost. MAX_BYTES_BILLED = 100 * 1024**3 # 100 GiB QUERY = f""" @@ -38,7 +37,7 @@ def _query_parameters(target_date: date, packages: list[str]) -> list: - """Build the BigQuery query parameters (parameterized to keep the scan pruned).""" + """Build the BigQuery query parameters.""" from google.cloud import bigquery return [ @@ -49,7 +48,7 @@ def _query_parameters(target_date: date, packages: list[str]) -> list: def _job_config(parameters: list) -> object: - """Query config: bind the parameters and cap bytes billed as a cost guard.""" + """Query config binding the parameters and capping bytes billed.""" from google.cloud import bigquery return bigquery.QueryJobConfig( diff --git a/tests/integration/test_raw_contract.py b/tests/integration/test_raw_contract.py index eb6303b..cd7e01e 100644 --- a/tests/integration/test_raw_contract.py +++ b/tests/integration/test_raw_contract.py @@ -1,9 +1,7 @@ """Integration test: the real sources normalize fixtures into the DuckDB `raw` schema. Drives the production `github_source` (HTTP mocked) and `pypi_source` (fake BigQuery -runner) from the recorded fixtures via `load_raw_fixtures` into a temporary -DuckDB and asserts the normalized table/column contract that the dbt staging models -depend on. +runner) from recorded fixtures, asserting the table/column contract dbt depends on. """ from collections.abc import Iterator