From f7d1e3e420e0d310a7b2f9393bba04d67aae500b Mon Sep 17 00:00:00 2001 From: Neukz Date: Fri, 26 Jun 2026 13:42:12 +0200 Subject: [PATCH] feat: migrate PyPI ingestion to BigQuery and add GitHub incremental loading --- .env.example | 8 +- dags/repolytics_daily.py | 40 +++-- dbt/models/marts/_marts__models.yml | 2 +- dbt/models/marts/fct_daily_downloads.sql | 11 +- dbt/models/staging/pypi/_pypi__models.yml | 11 +- dbt/models/staging/pypi/_pypi__sources.yml | 2 +- .../staging/pypi/stg_pypi__downloads.sql | 1 - docker-compose.yml | 4 + pyproject.toml | 1 + src/repolytics/config.py | 3 + src/repolytics/ingestion/github_source.py | 41 ++++- src/repolytics/ingestion/pipeline.py | 16 +- src/repolytics/ingestion/pypi_source.py | 103 +++++++++-- tests/fixtures/github_responses/issues.json | 2 + tests/fixtures/pypi_responses/downloads.json | 3 + tests/fixtures/pypi_responses/overall.json | 8 - tests/integration/test_github_incremental.py | 67 ++++++++ tests/integration/test_raw_contract.py | 21 ++- tests/support/warehouse.py | 29 ++-- tests/unit/test_pypi_source.py | 47 +++++ uv.lock | 162 ++++++++++++++++++ 21 files changed, 495 insertions(+), 87 deletions(-) create mode 100644 tests/fixtures/pypi_responses/downloads.json delete mode 100644 tests/fixtures/pypi_responses/overall.json create mode 100644 tests/integration/test_github_incremental.py create mode 100644 tests/unit/test_pypi_source.py diff --git a/.env.example b/.env.example index d685c0e..aa57bae 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,9 @@ # GitHub GITHUB_TOKEN=ghp_xxxxxxxxxxxxxxxxxxxx +# Google Cloud - PyPI download stats via the BigQuery public dataset +GCP_PROJECT=your-gcp-project-id + # Airflow -AIRFLOW_UID=50000 -# Generate: uv run python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" -FERNET_KEY=secret +AIRFLOW_UID=50000 # echo -e "AIRFLOW_UID=$(id -u)" +FERNET_KEY=secret # uv run python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" diff --git a/dags/repolytics_daily.py b/dags/repolytics_daily.py index d7ea0cb..930c91e 100644 --- a/dags/repolytics_daily.py +++ b/dags/repolytics_daily.py @@ -1,14 +1,17 @@ """Daily Repolytics pipeline: dlt ingestion -> dbt transform. -Ingestion runs as two independent tasks - GitHub and PyPI. Either can fail/retry -without touching the other. Both land into the DuckDB `raw` schema and are -upstream of the dbt transform. Cosmos renders the dbt project; `max_active_tasks=1` -serializes everything so no two tasks open the single-writer DuckDB file at once. +Ingestion runs as two independent tasks. PyPI is date-windowed: it queries the +BigQuery public dataset for the run's `data_interval_start`, so the DAG is +backfillable per day. GitHub uses dlt incremental cursors (global, not per-interval), +so it is skipped on historical backfill runs. Both land into the DuckDB `raw` schema +upstream of the dbt transform. Cosmos renders the dbt project; `max_active_tasks=1` + +`max_active_runs=1` serialize everything so no two tasks/runs open the single-writer +DuckDB file at once. """ import logging import os -from datetime import timedelta +from datetime import UTC, datetime, timedelta from pathlib import Path from airflow.sdk import dag, task @@ -50,7 +53,8 @@ def log_task_failure(context) -> None: dag_id="repolytics_daily", schedule="@daily", catchup=False, - max_active_tasks=1, # DuckDB is single-writer: never run two dbt tasks at once. + max_active_tasks=1, # DuckDB is single-writer: never run two dbt tasks at once + max_active_runs=1, # serialize runs so a backfill never overlaps DuckDB writers default_args={ "retries": 2, "retry_delay": timedelta(minutes=5), @@ -61,23 +65,39 @@ def log_task_failure(context) -> None: def repolytics_daily(): @task(multiple_outputs=False) def ingest_github() -> dict[str, int]: - """Run GitHub ingestion into the DuckDB `raw` dataset. + """Run GitHub incremental ingestion into the DuckDB `raw` dataset. + Skipped on historical backfill runs: the dlt cursor is global (not + per-interval), so replaying old intervals for GitHub is meaningless. Returns per-table row counts (pushed to XCom for the summary task). """ + from airflow.sdk import get_current_context + from repolytics.ingestion.pipeline import run_github + data_interval_end = get_current_context()["data_interval_end"] + if (datetime.now(UTC) - data_interval_end).days > 1: + logger.info( + "Skipping GitHub ingest for backfill interval ending %s", + data_interval_end, + ) + return {} return run_github() @task(multiple_outputs=False) def ingest_pypi() -> dict[str, int]: - """Run PyPI ingestion into the DuckDB `raw` dataset. + """Run PyPI ingestion for the run's data-interval day (BigQuery). - Returns per-table row counts (pushed to XCom for the summary task). + Uses `data_interval_start`, so each run loads exactly one day and + the DAG backfills cleanly per interval. Returns per-table row counts + (pushed to XCom for the summary task). """ + from airflow.sdk import get_current_context + from repolytics.ingestion.pipeline import run_pypi - return run_pypi() + target_date = get_current_context()["data_interval_start"].date() + return run_pypi(target_date=target_date) transform = DbtTaskGroup( group_id="transform", diff --git a/dbt/models/marts/_marts__models.yml b/dbt/models/marts/_marts__models.yml index f67ec0d..d9733f6 100644 --- a/dbt/models/marts/_marts__models.yml +++ b/dbt/models/marts/_marts__models.yml @@ -383,7 +383,7 @@ models: - name: fct_daily_downloads description: > - One row per package per day (PyPI 'without_mirrors' downloads). Incremental + One row per package per day (PyPI non-mirror downloads). Incremental (delete+insert on download_key). data_tests: - dbt_expectations.expect_table_row_count_to_be_between: diff --git a/dbt/models/marts/fct_daily_downloads.sql b/dbt/models/marts/fct_daily_downloads.sql index 8b64e03..d67cde0 100644 --- a/dbt/models/marts/fct_daily_downloads.sql +++ b/dbt/models/marts/fct_daily_downloads.sql @@ -1,10 +1,8 @@ -- PyPI daily download fact: one row per package per day. `repository_key` is resolved -- through the `projects` seed (package -> repo) and the SCD2 dim_repositories half-open --- range; it is nullable for packages with no mapped/ingested repo. --- Filtered to the 'without_mirrors' overall time-series category to avoid double --- counting 'with_mirrors' and to exclude the recent-endpoint 'last_*' aggregates. --- Incremental (delete+insert on download_key): only processes days at or after the --- latest download_date already loaded; the unique key keeps it idempotent. +-- range; it is nullable for packages with no mapped/ingested repo. Incremental +-- (delete+insert on download_key): only processes days at or after the latest +-- download_date already loaded; the unique key keeps it idempotent. {{ config( @@ -17,10 +15,9 @@ with downloads as ( select * from {{ ref('stg_pypi__downloads') }} - where category = 'without_mirrors' {% if is_incremental() %} -- date_key is the only date column on the target table; compare the day's key. - and {{ date_key('download_date') }} >= (select max(date_key) from {{ this }}) + where {{ date_key('download_date') }} >= (select max(date_key) from {{ this }}) {% endif %} ) diff --git a/dbt/models/staging/pypi/_pypi__models.yml b/dbt/models/staging/pypi/_pypi__models.yml index 42466de..9b4473a 100644 --- a/dbt/models/staging/pypi/_pypi__models.yml +++ b/dbt/models/staging/pypi/_pypi__models.yml @@ -2,17 +2,10 @@ version: 2 models: - name: stg_pypi__downloads - description: One cleaned row per package/category/day download count. + description: One cleaned row per package/day non-mirror download count. columns: - name: package data_tests: [not_null] - - name: category - description: PyPI overall time-series category. - data_tests: - - not_null - - accepted_values: - arguments: - values: [with_mirrors, without_mirrors] - name: download_date data_tests: [not_null] - name: download_count @@ -20,4 +13,4 @@ models: data_tests: - dbt_utils.unique_combination_of_columns: arguments: - combination_of_columns: [package, category, download_date] + combination_of_columns: [package, download_date] diff --git a/dbt/models/staging/pypi/_pypi__sources.yml b/dbt/models/staging/pypi/_pypi__sources.yml index 73ed8d0..eb96bdc 100644 --- a/dbt/models/staging/pypi/_pypi__sources.yml +++ b/dbt/models/staging/pypi/_pypi__sources.yml @@ -2,7 +2,7 @@ version: 2 sources: - name: pypi - description: PyPI Stats data loaded by dlt; one row per package/day/category. + description: PyPI download data loaded by dlt from BigQuery; one row per package/day. schema: raw loaded_at_field: _loaded_at freshness: diff --git a/dbt/models/staging/pypi/stg_pypi__downloads.sql b/dbt/models/staging/pypi/stg_pypi__downloads.sql index 058187c..0e776d6 100644 --- a/dbt/models/staging/pypi/stg_pypi__downloads.sql +++ b/dbt/models/staging/pypi/stg_pypi__downloads.sql @@ -6,7 +6,6 @@ with downloads as ( select _package as package, - category, date::date as download_date, downloads as download_count, _loaded_at diff --git a/docker-compose.yml b/docker-compose.yml index f3c2a03..c1ff487 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,6 +79,8 @@ x-airflow-common: DUCKDB_PATH: /opt/airflow/data/warehouse/repolytics.duckdb # dbt project path DBT_PROJECT_DIR: /opt/airflow/dbt + # BigQuery (PyPI download stats): ADC file mounted from the host (see volumes). + GOOGLE_APPLICATION_CREDENTIALS: /opt/airflow/gcloud/adc.json volumes: - ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs @@ -90,6 +92,8 @@ x-airflow-common: - ${AIRFLOW_PROJ_DIR:-.}/data:/opt/airflow/data # repolytics package source (src-layout) - ${AIRFLOW_PROJ_DIR:-.}/src:/opt/airflow/project/src + # Google Cloud ADC + - ${HOME}/.config/gcloud/application_default_credentials.json:/opt/airflow/gcloud/adc.json:ro user: "${AIRFLOW_UID:-50000}:0" depends_on: &airflow-common-depends-on diff --git a/pyproject.toml b/pyproject.toml index ac571d7..f697a16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ requires-python = ">=3.13" dependencies = [ "dlt[duckdb]>=1.28.0", "duckdb>=1.5.3", + "google-cloud-bigquery>=3.42.1", "pydantic-settings>=2.14.1", ] diff --git a/src/repolytics/config.py b/src/repolytics/config.py index 2fe2a20..37f6a0a 100644 --- a/src/repolytics/config.py +++ b/src/repolytics/config.py @@ -26,6 +26,9 @@ class Settings(BaseSettings): # DuckDB duckdb_path: Path = Path("data/warehouse/repolytics.duckdb") + # Google Cloud project that BigQuery PyPI-download jobs are billed to. + gcp_project: str | None = None + # Project list (repo <-> package) - also loaded by dbt as the `projects` seed. projects_file: Path = Path("dbt/seeds/projects.csv") diff --git a/src/repolytics/ingestion/github_source.py b/src/repolytics/ingestion/github_source.py index 5a16b79..5320cf6 100644 --- a/src/repolytics/ingestion/github_source.py +++ b/src/repolytics/ingestion/github_source.py @@ -44,10 +44,19 @@ def repositories() -> Iterator[dict]: yield stamp(_repo=repo)(response.json()) @dlt.resource(name="commits", write_disposition="merge", primary_key="sha") - def commits() -> Iterator[dict]: + def commits( + updated: dlt.sources.incremental[str] = dlt.sources.incremental( # noqa: B008 + "commit.author.date" + ), + ) -> Iterator[dict]: + # `since` (server-side, on commit date) bounds the fetch; dlt filters + merge + # keep it idempotent. The cursor is global across repos and commit dates can be + # backdated (rebases), so a backdated commit pushed today may be missed - the + # same trade-off the fct_commits watermark documents. + params = {"since": updated.last_value} if updated.last_value else {} for repo in repos: owner, name = repo.split("/") - yield from _paginate(f"/repos/{owner}/{name}/commits", repo) + yield from _paginate(f"/repos/{owner}/{name}/commits", repo, params) @dlt.resource( name="issues", @@ -57,22 +66,38 @@ def commits() -> Iterator[dict]: # PR-shaped issues, so the staging PR filter never references a missing column. columns={"pull_request__url": {"data_type": "text", "nullable": True}}, ) - def issues() -> Iterator[dict]: + def issues( + updated: dlt.sources.incremental[str] = dlt.sources.incremental( # noqa: B008 + "updated_at" + ), + ) -> Iterator[dict]: + # `since` filters server-side on updated_at; sort asc so the cursor advances + # monotonically. Global cursor across repos (see commits caveat). + params = {"state": "all", "sort": "updated", "direction": "asc"} + if updated.last_value: + params["since"] = updated.last_value for repo in repos: owner, name = repo.split("/") - yield from _paginate( - f"/repos/{owner}/{name}/issues", repo, {"state": "all"} - ) + yield from _paginate(f"/repos/{owner}/{name}/issues", repo, params) @dlt.resource( name="pull_requests", write_disposition="merge", primary_key=["_repo", "number"], # PR number is unique per repo ) - def pull_requests() -> Iterator[dict]: + def pull_requests( + updated: dlt.sources.incremental[str] = dlt.sources.incremental( # noqa: B008 + "updated_at" + ), + ) -> Iterator[dict]: + # The /pulls endpoint has no `since`, so we can't bound the fetch server-side + # and can't use dlt's row_order early-exit either (it assumes one monotonic + # stream, but looping repos makes the updated_at sequence saw-tooth). We page + # the PR list and let dlt's cursor + merge drop/upsert unchanged rows. + params = {"state": "all", "sort": "updated", "direction": "desc"} for repo in repos: owner, name = repo.split("/") - yield from _paginate(f"/repos/{owner}/{name}/pulls", repo, {"state": "all"}) + yield from _paginate(f"/repos/{owner}/{name}/pulls", repo, params) @dlt.resource(name="releases", write_disposition="merge", primary_key="id") def releases() -> Iterator[dict]: diff --git a/src/repolytics/ingestion/pipeline.py b/src/repolytics/ingestion/pipeline.py index 88d0537..e751c95 100644 --- a/src/repolytics/ingestion/pipeline.py +++ b/src/repolytics/ingestion/pipeline.py @@ -1,6 +1,7 @@ """dlt pipelines wiring GitHub and PyPI ingestion into the DuckDB `raw` dataset.""" import logging +from datetime import UTC, date, datetime, timedelta import dlt @@ -51,11 +52,14 @@ def run_github(settings: Settings | None = None) -> dict[str, int]: return _table_row_counts(pipeline) -def run_pypi(settings: Settings | None = None) -> dict[str, int]: - """Run PyPI ingestion into the configured DuckDB warehouse. +def run_pypi( + settings: Settings | None = None, target_date: date | None = None +) -> dict[str, int]: + """Run PyPI ingestion for a single day into the configured DuckDB warehouse. - No-ops (logs and returns `{}`) when no packages are configured. Returns the - per-table row counts loaded in this run. + Queries the BigQuery public dataset for `target_date` (defaults to yesterday + UTC, the most recent complete partition). No-ops (logs and returns `{}`) when + no packages are configured. Returns the per-table row counts loaded in this run. """ settings = settings or get_settings() @@ -64,9 +68,11 @@ def run_pypi(settings: Settings | None = None) -> dict[str, int]: logger.warning("No packages to ingest - check %s", settings.projects_file) return {} + target_date = target_date or (datetime.now(UTC).date() - timedelta(days=1)) settings.duckdb_path.parent.mkdir(parents=True, exist_ok=True) pipeline = build_pipeline(settings) - logger.info("PyPI ingestion complete: %s", pipeline.run(pypi_source(packages))) + source = pypi_source(packages, target_date, project=settings.gcp_project) + logger.info("PyPI ingestion complete (%s): %s", target_date, pipeline.run(source)) return _table_row_counts(pipeline) diff --git a/src/repolytics/ingestion/pypi_source.py b/src/repolytics/ingestion/pypi_source.py index 6edd5e6..e2fe7f8 100644 --- a/src/repolytics/ingestion/pypi_source.py +++ b/src/repolytics/ingestion/pypi_source.py @@ -1,36 +1,105 @@ -"""dlt source for the PyPI Stats API. +"""dlt source for PyPI download stats from the Google BigQuery public dataset. -Fetches the per-package overall download time series and yields one row per -day/category, stamped with `_package` + `_loaded_at`. +Queries the day-partitioned `bigquery-public-data.pypi.file_downloads` table for +a single date (partition-pruned, so each run scans only that day), aggregating +non-mirror downloads per package. Date windowing is the increment - re-running a +date is idempotent via `write_disposition="merge"` on `(_package, date)`. """ -import time -from collections.abc import Iterator +from collections.abc import Callable, Iterable, Iterator, Mapping +from datetime import date import dlt -from dlt.sources.helpers.rest_client import RESTClient from repolytics.ingestion._meta import stamp -BASE_URL = "https://pypistats.org/api" +TABLE = "bigquery-public-data.pypi.file_downloads" + +# Bulk-mirror installers excluded so counts reflect real installs, matching pypistats' +# "without_mirrors" definition. +# See: https://pypistats.org/faqs#what-is-the-difference-between-without_mirrors-and-with_mirrors +MIRROR_INSTALLERS = ["bandersnatch", "z3c.pypimirror", "Artifactory", "devpi"] + +# Hard ceiling on bytes scanned per query. A partition-pruned single-day query stays +# well under this, so hitting it means the date filter was lost +MAX_BYTES_BILLED = 100 * 1024**3 # 100 GiB + +QUERY = f""" +SELECT file.project AS package, COUNT(*) AS downloads +FROM `{TABLE}` +WHERE DATE(timestamp) = @target_date + AND file.project IN UNNEST(@packages) + AND COALESCE(details.installer.name, '') NOT IN UNNEST(@mirror_installers) +GROUP BY file.project +""" + +# (sql, query_parameters) -> rows, each row mapping `package` and `downloads`. +QueryRunner = Callable[[str, list], Iterable[Mapping]] + + +def _query_parameters(target_date: date, packages: list[str]) -> list: + """Build the BigQuery query parameters (parameterized to keep the scan pruned).""" + from google.cloud import bigquery + + return [ + bigquery.ScalarQueryParameter("target_date", "DATE", target_date), + bigquery.ArrayQueryParameter("packages", "STRING", packages), + bigquery.ArrayQueryParameter("mirror_installers", "STRING", MIRROR_INSTALLERS), + ] + + +def _job_config(parameters: list) -> object: + """Query config: bind the parameters and cap bytes billed as a cost guard.""" + from google.cloud import bigquery + + return bigquery.QueryJobConfig( + query_parameters=parameters, + maximum_bytes_billed=MAX_BYTES_BILLED, + ) + + +def _default_runner(project: str | None) -> QueryRunner: + """Run the query against BigQuery, billing jobs to `project`.""" + + def run(sql: str, parameters: list) -> Iterable[Mapping]: + from google.cloud import bigquery + + client = bigquery.Client(project=project) + return client.query(sql, job_config=_job_config(parameters)).result() + + return run @dlt.source(name="pypi") -def pypi_source(packages: list[str], *, min_interval: float = 1.0) -> object: - """dlt source yielding PyPI overall downloads for each package.""" - client = RESTClient(base_url=BASE_URL) +def pypi_source( + packages: list[str], + target_date: date, + *, + project: str | None = None, + query_runner: QueryRunner | None = None, +) -> object: + """dlt source yielding one daily non-mirror download count per package. + + `target_date` selects the BigQuery day partition. `query_runner` is injectable + for offline tests; by default it runs against BigQuery using `project` for billing. + """ + runner = query_runner or _default_runner(project) @dlt.resource( name="downloads", write_disposition="merge", - primary_key=["_package", "category", "date"], + primary_key=["_package", "date"], ) def downloads() -> Iterator[dict]: - for package in packages: - response = client.get(f"/packages/{package}/overall") - response.raise_for_status() - apply = stamp(_package=package) - yield from (apply(row) for row in response.json()["data"]) - time.sleep(min_interval) # courtesy delay between packages + apply = stamp() # adds _loaded_at only; package is a real column + iso = target_date.isoformat() + for row in runner(QUERY, _query_parameters(target_date, packages)): + yield apply( + { + "_package": row["package"], + "date": iso, + "downloads": row["downloads"], + } + ) return downloads diff --git a/tests/fixtures/github_responses/issues.json b/tests/fixtures/github_responses/issues.json index 907b6bc..30b61e4 100644 --- a/tests/fixtures/github_responses/issues.json +++ b/tests/fixtures/github_responses/issues.json @@ -5,6 +5,7 @@ "state": "closed", "user": {"login": "carol", "id": 3}, "created_at": "2024-01-01T00:00:00Z", + "updated_at": "2024-01-05T00:00:00Z", "closed_at": "2024-01-05T00:00:00Z", "comments": 4, "labels": [{"id": 1, "name": "bug", "color": "d73a4a"}] @@ -15,6 +16,7 @@ "state": "open", "user": {"login": "dave", "id": 4}, "created_at": "2024-01-02T00:00:00Z", + "updated_at": "2024-01-02T00:00:00Z", "closed_at": null, "comments": 0, "labels": [], diff --git a/tests/fixtures/pypi_responses/downloads.json b/tests/fixtures/pypi_responses/downloads.json new file mode 100644 index 0000000..a775217 --- /dev/null +++ b/tests/fixtures/pypi_responses/downloads.json @@ -0,0 +1,3 @@ +[ + {"package": "httpx", "downloads": 12000} +] diff --git a/tests/fixtures/pypi_responses/overall.json b/tests/fixtures/pypi_responses/overall.json deleted file mode 100644 index f97e647..0000000 --- a/tests/fixtures/pypi_responses/overall.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "data": [ - {"category": "without_mirrors", "date": "2024-01-01", "downloads": 12000}, - {"category": "without_mirrors", "date": "2024-01-02", "downloads": 13500} - ], - "package": "polars", - "type": "overall_downloads" -} diff --git a/tests/integration/test_github_incremental.py b/tests/integration/test_github_incremental.py new file mode 100644 index 0000000..7944b53 --- /dev/null +++ b/tests/integration/test_github_incremental.py @@ -0,0 +1,67 @@ +"""Integration test: GitHub incremental cursors bound later runs with `since`. + +Runs the real `github_source` (HTTP mocked from fixtures) into a temp DuckDB twice and +inspects the outgoing requests: the first run fetches full history, the second sends a +`since` bound once the cursor has advanced. Repositories stay a full fetch either run. +""" + +from pathlib import Path +from urllib.parse import parse_qs, urlparse + +import dlt +import responses + +from repolytics.ingestion.github_source import github_source +from tests.support.warehouse import REPO, register + + +def _run(duckdb_path: Path, pipelines_dir: Path) -> None: + pipeline = dlt.pipeline( + pipeline_name="gh_incremental_test", + destination=dlt.destinations.duckdb(str(duckdb_path)), + dataset_name="raw", + pipelines_dir=str(pipelines_dir), # isolate cursor state per test + ) + pipeline.run(github_source([REPO], token="t")) + + +def _query_for(rsps: responses.RequestsMock, suffix: str) -> dict: + url = next( + c.request.url + for c in rsps.calls + if urlparse(c.request.url).path.endswith(suffix) + ) + return parse_qs(urlparse(url).query) + + +def _repo_root_called(rsps: responses.RequestsMock) -> bool: + owner, name = REPO.split("/") + root = f"/repos/{owner}/{name}" + return any(urlparse(c.request.url).path.endswith(root) for c in rsps.calls) + + +def test_incremental_wiring(tmp_duckdb_path: Path, tmp_path: Path) -> None: + pdir = tmp_path / "dlt" + + # First run: no prior state -> commits fetch full history (no `since`); PRs are + # sorted by updated desc; repositories are fetched in full. + with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps: + register(rsps) + _run(tmp_duckdb_path, pdir) + commits_q = _query_for(rsps, "/commits") + pulls_q = _query_for(rsps, "/pulls") + assert _repo_root_called(rsps) + + assert "since" not in commits_q + assert pulls_q["sort"] == ["updated"] + assert pulls_q["direction"] == ["desc"] + + # Second run: the commit-date cursor advanced, so `since` now bounds the fetch, + # and repositories are still fetched in full (not incremental). + with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps: + register(rsps) + _run(tmp_duckdb_path, pdir) + commits_q2 = _query_for(rsps, "/commits") + assert _repo_root_called(rsps) + + assert "since" in commits_q2 diff --git a/tests/integration/test_raw_contract.py b/tests/integration/test_raw_contract.py index bd52376..eb6303b 100644 --- a/tests/integration/test_raw_contract.py +++ b/tests/integration/test_raw_contract.py @@ -1,8 +1,9 @@ """Integration test: the real sources normalize fixtures into the DuckDB `raw` schema. -Drives the production `github_source` / `pypi_source` (HTTP mocked from the recorded -fixtures via `tests.support.warehouse.load_raw_fixtures`) into a temporary DuckDB and -asserts the normalized table/column contract that the dbt staging models depend on. +Drives the production `github_source` (HTTP mocked) and `pypi_source` (fake BigQuery +runner) from the recorded fixtures via `load_raw_fixtures` into a temporary +DuckDB and asserts the normalized table/column contract that the dbt staging models +depend on. """ from collections.abc import Iterator @@ -66,6 +67,18 @@ def test_metadata_columns_present(loaded_db: duckdb.DuckDBPyConnection) -> None: assert package == PACKAGE +def test_downloads_contract(loaded_db: duckdb.DuckDBPyConnection) -> None: + # One non-mirror count per package/day: the columns the staging model reads. + cols = { + r[0] + for r in loaded_db.execute( + "select column_name from information_schema.columns " + "where table_schema = 'raw' and table_name = 'downloads'" + ).fetchall() + } + assert {"_package", "date", "downloads", "_loaded_at"} <= cols + + def test_pull_request_marker_column_exists( loaded_db: duckdb.DuckDBPyConnection, ) -> None: @@ -87,4 +100,4 @@ def count(table: str) -> int: assert count("commits") == 2 assert count("issues") == 2 # includes the PR-shaped issue (staging filters it) - assert count("downloads") == 2 + assert count("downloads") == 1 # one package, one day from the BigQuery fixture diff --git a/tests/support/warehouse.py b/tests/support/warehouse.py index 6b029dd..cd29b78 100644 --- a/tests/support/warehouse.py +++ b/tests/support/warehouse.py @@ -1,11 +1,13 @@ """Load the recorded API fixtures into a DuckDB `raw` dataset via the real sources. -Run standalone (used by CI to seed a warehouse before `dbt build`):: +Run standalone (used by CI to seed a warehouse before `dbt build`): DUCKDB_PATH=/tmp/ci.duckdb python -m tests.support.warehouse """ import json import os +from collections.abc import Iterable, Mapping +from datetime import date from pathlib import Path import dlt @@ -17,20 +19,20 @@ FIXTURES = Path(__file__).parent.parent / "fixtures" REPO = "encode/httpx" PACKAGE = "httpx" - +TARGET_DATE = date(2024, 1, 1) GITHUB_API = "https://api.github.com" -PYPI_API = "https://pypistats.org/api" def _json(rel: str) -> object: return json.loads((FIXTURES / rel).read_text(encoding="utf-8")) -def _register(rsps: responses.RequestsMock) -> None: - """Stub every endpoint the real sources hit, one single-page response each. +def register(rsps: responses.RequestsMock) -> None: + """Stub every GitHub endpoint the real source hits, one single-page response each. No `Link` header is set, so dlt's `RESTClient.paginate` treats each list - response as a single page and stops. + response as a single page and stops. Stubs registered without a query string + match regardless of the incremental `since`/`sort` params the source adds. """ owner, name = REPO.split("/") base = f"{GITHUB_API}/repos/{owner}/{name}" @@ -39,14 +41,15 @@ def _register(rsps: responses.RequestsMock) -> None: rsps.get(f"{base}/issues", json=_json("github_responses/issues.json")) rsps.get(f"{base}/pulls", json=_json("github_responses/pulls.json")) rsps.get(f"{base}/releases", json=_json("github_responses/releases.json")) - rsps.get( - f"{PYPI_API}/packages/{PACKAGE}/overall", - json=_json("pypi_responses/overall.json"), - ) + + +def _fake_bq_runner(_sql: str, _parameters: list) -> Iterable[Mapping]: + """Offline stand-in for the BigQuery client: returns recorded result rows.""" + return _json("pypi_responses/downloads.json") def load_raw_fixtures(duckdb_path: Path) -> None: - """Run the real GitHub + PyPI sources against the fixtures into `raw`.""" + """Run the real GitHub source (HTTP mocked) + BigQuery PyPI source (fake runner).""" pipeline = dlt.pipeline( pipeline_name="repolytics_fixtures", destination=dlt.destinations.duckdb(str(duckdb_path)), @@ -55,9 +58,9 @@ def load_raw_fixtures(duckdb_path: Path) -> None: # assert_all_requests_are_fired=False: dlt may not re-hit every stub on a no-op # pass, and we only care that the calls it does make are answered from fixtures. with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps: - _register(rsps) + register(rsps) pipeline.run(github_source([REPO], token="fixture-token")) - pipeline.run(pypi_source([PACKAGE], min_interval=0)) + pipeline.run(pypi_source([PACKAGE], TARGET_DATE, query_runner=_fake_bq_runner)) if __name__ == "__main__": diff --git a/tests/unit/test_pypi_source.py b/tests/unit/test_pypi_source.py new file mode 100644 index 0000000..581481e --- /dev/null +++ b/tests/unit/test_pypi_source.py @@ -0,0 +1,47 @@ +"""Unit tests for the BigQuery PyPI source (offline, with a fake query runner).""" + +from datetime import date + +from repolytics.ingestion import pypi_source as bq + + +def test_query_is_partition_pruned() -> None: + # Pruning on the date partition keeps the scan (and cost) tiny. + assert "DATE(timestamp) = @target_date" in bq.QUERY + assert "file.project IN UNNEST(@packages)" in bq.QUERY + assert "NOT IN UNNEST(@mirror_installers)" in bq.QUERY + + +def test_job_config_caps_bytes_billed() -> None: + # A cost guard: a mispruned (full-table) query fails instead of running up a bill. + cfg = bq._job_config(bq._query_parameters(date(2024, 1, 1), ["httpx"])) + assert cfg.maximum_bytes_billed == bq.MAX_BYTES_BILLED + + +def test_query_parameters_are_typed() -> None: + params = bq._query_parameters(date(2024, 1, 1), ["httpx", "polars"]) + by_name = {p.name: p for p in params} + + assert by_name["target_date"].value == date(2024, 1, 1) + assert by_name["packages"].values == ["httpx", "polars"] + assert by_name["mirror_installers"].values == bq.MIRROR_INSTALLERS + + +def test_rows_are_mapped_and_stamped() -> None: + captured: dict = {} + + def fake_runner(sql: str, parameters: list) -> list[dict]: + captured["sql"] = sql + captured["parameters"] = parameters + return [{"package": "httpx", "downloads": 12000}] + + source = bq.pypi_source(["httpx"], date(2024, 1, 1), query_runner=fake_runner) + rows = list(source.downloads) + + assert captured["sql"] == bq.QUERY + assert len(rows) == 1 + row = rows[0] + assert row["_package"] == "httpx" + assert row["date"] == "2024-01-01" + assert row["downloads"] == 12000 + assert "_loaded_at" in row # stamped provenance diff --git a/uv.lock b/uv.lock index fdb2b70..46bb55e 100644 --- a/uv.lock +++ b/uv.lock @@ -1060,6 +1060,107 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/96/147a2771ab655b9353781fb2f95c94eaf1d8576dccc991c1a61d0d355067/giturlparse-0.15.0-py2.py3-none-any.whl", hash = "sha256:76d2e6983b037356ab99b30683e533ac3db96409b68e2163a20fc3aff6446f10", size = 16683, upload-time = "2026-06-16T07:28:55.184Z" }, ] +[[package]] +name = "google-api-core" +version = "2.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "googleapis-common-protos" }, + { name = "proto-plus" }, + { name = "protobuf" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/22/155cadf1d49272a9cf48f3168c0f3874fa13397297e611a5ea00cd093880/google_api_core-2.31.0.tar.gz", hash = "sha256:2be84ee0f584c48e6bde1b36766e23348b361fb7e55e56135fc76ce1c397f9c2", size = 176492, upload-time = "2026-06-03T14:52:17.257Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/40/9bdbb60b03a332bd45acb8703da08bbc27d991d35286b62e42acc86d243a/google_api_core-2.31.0-py3-none-any.whl", hash = "sha256:ef79fb3784c71cbac89cbd03301ba0c8fb8ad2aa95d7f9204dd9628f7adf59ab", size = 173102, upload-time = "2026-06-03T14:51:26.729Z" }, +] + +[package.optional-dependencies] +grpc = [ + { name = "grpcio" }, + { name = "grpcio-status" }, +] + +[[package]] +name = "google-auth" +version = "2.55.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyasn1-modules" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/1c/70b23fc52b2bb3c70b379f3bd05c4a60ab3a873e30c6bd21c57e0154848a/google_auth-2.55.0.tar.gz", hash = "sha256:fcd3a130f575fa36403d38774af1c64a4fbfbca09215f0589d2372b5119697cb", size = 349379, upload-time = "2026-06-15T22:33:16.466Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/71/c0321dc6d63d99946da45f7c06299b934e4f7f7da5c4f14d101bcb39adf1/google_auth-2.55.0-py3-none-any.whl", hash = "sha256:a17cef9dedf98c4ebae2fb0c48c8f75952c877cbc2efe09f329ef16c2783d88a", size = 252400, upload-time = "2026-06-15T22:33:14.992Z" }, +] + +[package.optional-dependencies] +pyopenssl = [ + { name = "pyopenssl" }, +] + +[[package]] +name = "google-cloud-bigquery" +version = "3.42.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-auth", extra = ["pyopenssl"] }, + { name = "google-cloud-core" }, + { name = "google-resumable-media" }, + { name = "packaging" }, + { name = "python-dateutil" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/15/a2/be1cffbb1a9894ecf394ab0096b113ae9e4f73686595d906a9f3082b07c2/google_cloud_bigquery-3.42.1.tar.gz", hash = "sha256:3c6878f424fc2b21f0bb414ca7262abd008465575a7b7c44c552105278a18914", size = 515411, upload-time = "2026-06-22T23:21:05.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/1e/8b098aeb69ef87de5195d076ce6d82002c3ebf1920373d386ca7b6e63cfa/google_cloud_bigquery-3.42.1-py3-none-any.whl", hash = "sha256:159143c1b7248858f35549c09efea5d031c2883145623419ea7c505a50c0899a", size = 263814, upload-time = "2026-06-22T23:18:42.313Z" }, +] + +[[package]] +name = "google-cloud-core" +version = "2.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/dd/1eef226e470369b26824a505c34482c0b493bc35fe8e0c6b003b5feca21a/google_cloud_core-2.6.0.tar.gz", hash = "sha256:e76149739f90fac1fc6757c09f47eaccb3145b54adbd7759b0f7c4b235f46c83", size = 36001, upload-time = "2026-05-07T08:04:04.124Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/4a/98da8930ab109c73d9a5d13782a9ebb81ea8c111f6d534a567b71d23e52b/google_cloud_core-2.6.0-py3-none-any.whl", hash = "sha256:6d63ac8e5eca6d9e4319d0a1e2265fadcd7f1049904378caecfa01cf52dd869e", size = 29390, upload-time = "2026-05-07T08:02:34.672Z" }, +] + +[[package]] +name = "google-crc32c" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/db/000f15b41724589b0e7bc24bc7a8967898d8d3bc8caf64c513d91ef1f6c0/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b", size = 31297, upload-time = "2025-12-16T00:23:20.709Z" }, + { url = "https://files.pythonhosted.org/packages/d7/0d/8ebed0c39c53a7e838e2a486da8abb0e52de135f1b376ae2f0b160eb4c1a/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27", size = 30867, upload-time = "2025-12-16T00:43:14.628Z" }, + { url = "https://files.pythonhosted.org/packages/ce/42/b468aec74a0354b34c8cbf748db20d6e350a68a2b0912e128cabee49806c/google_crc32c-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa", size = 33344, upload-time = "2025-12-16T00:40:24.742Z" }, + { url = "https://files.pythonhosted.org/packages/1c/e8/b33784d6fc77fb5062a8a7854e43e1e618b87d5ddf610a88025e4de6226e/google_crc32c-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8", size = 33694, upload-time = "2025-12-16T00:40:25.505Z" }, + { url = "https://files.pythonhosted.org/packages/92/b1/d3cbd4d988afb3d8e4db94ca953df429ed6db7282ed0e700d25e6c7bfc8d/google_crc32c-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f", size = 34435, upload-time = "2025-12-16T00:35:22.107Z" }, + { url = "https://files.pythonhosted.org/packages/21/88/8ecf3c2b864a490b9e7010c84fd203ec8cf3b280651106a3a74dd1b0ca72/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:e6584b12cb06796d285d09e33f63309a09368b9d806a551d8036a4207ea43697", size = 31301, upload-time = "2025-12-16T00:24:48.527Z" }, + { url = "https://files.pythonhosted.org/packages/36/c6/f7ff6c11f5ca215d9f43d3629163727a272eabc356e5c9b2853df2bfe965/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651", size = 30868, upload-time = "2025-12-16T00:48:12.163Z" }, + { url = "https://files.pythonhosted.org/packages/56/15/c25671c7aad70f8179d858c55a6ae8404902abe0cdcf32a29d581792b491/google_crc32c-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2", size = 33381, upload-time = "2025-12-16T00:40:26.268Z" }, + { url = "https://files.pythonhosted.org/packages/42/fa/f50f51260d7b0ef5d4898af122d8a7ec5a84e2984f676f746445f783705f/google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21", size = 33734, upload-time = "2025-12-16T00:40:27.028Z" }, + { url = "https://files.pythonhosted.org/packages/08/a5/7b059810934a09fb3ccb657e0843813c1fee1183d3bc2c8041800374aa2c/google_crc32c-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2", size = 34878, upload-time = "2025-12-16T00:35:23.142Z" }, +] + +[[package]] +name = "google-resumable-media" +version = "2.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-crc32c" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/f8/1ca5781d6be9cb9f73f7d40f4958c4bd1226a60598e3e39e1d6aaf838c4b/google_resumable_media-2.10.0.tar.gz", hash = "sha256:e324bc9d0fdae4c52a08ae90456edc4e71ece858399e1217ac0eb3a51d6bc6ee", size = 2164570, upload-time = "2026-06-03T16:14:26.103Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/d8/00c6854ac1512bb9eaf13bd3f8f28222f7674947fc510a4ff7616f2efc80/google_resumable_media-2.10.0-py3-none-any.whl", hash = "sha256:88152884bee37b2bf36a0ab81ad8c7fd12212c9803dd981d77c1b35b02d34e7c", size = 81533, upload-time = "2026-06-03T16:13:12.51Z" }, +] + [[package]] name = "googleapis-common-protos" version = "1.75.0" @@ -1174,6 +1275,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ff/b0/4af731ff7492c68a96e4c71bfd0f4590acde92b31c6fe4894e6465c10ff6/grpcio-1.81.1-cp314-cp314-win_amd64.whl", hash = "sha256:3768a5ff1b2125e6f552e561b6b2dca0e64982d8949689b4df145cf8b98d7821", size = 5070275, upload-time = "2026-06-11T12:46:48.486Z" }, ] +[[package]] +name = "grpcio-status" +version = "1.81.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/32/26/0aa9168c87882381fd810d140c279a2490ed6aee655f0515d6f56c5ca404/grpcio_status-1.81.1.tar.gz", hash = "sha256:9389a03e746017b10f0630c064289201458f3ce01f5d7ef4b0bebc1ef6cf82ad", size = 13923, upload-time = "2026-06-11T12:58:48.636Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/5e/5abfec5f7e89d3b7993d57cfb025ca5f968a2c18656d7fcda2b6919440b9/grpcio_status-1.81.1-py3-none-any.whl", hash = "sha256:08072fa9995f4a95c647fc6f4f85e2411573d00087bcabdf30f260114338f232", size = 14638, upload-time = "2026-06-11T12:58:31.982Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -1979,6 +2094,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/58/35da89ee790598a0700ea49b2a66594140f44dec458c07e8e3d4979137fc/ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce", size = 49567, upload-time = "2018-02-15T19:01:27.172Z" }, ] +[[package]] +name = "proto-plus" +version = "1.28.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/56/e647b0c675392d2da368da7b6f158f7368b18542fd6f7d7400a2f39de000/proto_plus-1.28.0.tar.gz", hash = "sha256:38e5696342835b08fc116f30a25665b29531cda9d5d5643e9b81fc312385abd9", size = 57221, upload-time = "2026-05-07T08:04:50.811Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/20/b122d4626976acb81132036d2ad1bb35a1a8775fceb837ec30964622516a/proto_plus-1.28.0-py3-none-any.whl", hash = "sha256:a630604310899e73c59ec302e5765c058d412b2f090b9c79c8822589f14955b8", size = 50410, upload-time = "2026-05-07T08:03:31.962Z" }, +] + [[package]] name = "protobuf" version = "6.33.6" @@ -2022,6 +2149,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, ] +[[package]] +name = "pyasn1" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, +] + [[package]] name = "pycparser" version = "3.0" @@ -2156,6 +2304,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, ] +[[package]] +name = "pyopenssl" +version = "26.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/b7/da07bae88f5a9506b4def6f2f4903cf4c3b8831e560dba8fa18ca08f758f/pyopenssl-26.3.0.tar.gz", hash = "sha256:589de7fae1c9ea670d18422ed00fc04da787bbde8e1454aea872aa57b49ad341", size = 182024, upload-time = "2026-06-12T20:28:07.458Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/18/1dd71c9b43192ab83f1d531ad6002dc81108ac36c475f79fb7a295abe2f4/pyopenssl-26.3.0-py3-none-any.whl", hash = "sha256:46367f8f66b92271e6d218da9c87607e1ef5a0bc5c8dea5bb3db82f395c385a3", size = 56008, upload-time = "2026-06-12T20:28:05.999Z" }, +] + [[package]] name = "pytest" version = "9.1.0" @@ -2367,6 +2527,7 @@ source = { editable = "." } dependencies = [ { name = "dlt", extra = ["duckdb"] }, { name = "duckdb" }, + { name = "google-cloud-bigquery" }, { name = "pydantic-settings" }, ] @@ -2389,6 +2550,7 @@ orchestration = [ requires-dist = [ { name = "dlt", extras = ["duckdb"], specifier = ">=1.28.0" }, { name = "duckdb", specifier = ">=1.5.3" }, + { name = "google-cloud-bigquery", specifier = ">=3.42.1" }, { name = "pydantic-settings", specifier = ">=2.14.1" }, ]