From c8958eb62e4c5864f0bfe462c8378abacb09d092 Mon Sep 17 00:00:00 2001 From: Neukz Date: Wed, 17 Jun 2026 09:24:49 +0200 Subject: [PATCH] refactor: replace custom ingestion/loading with dlt --- .env.example | 5 - .gitignore | 5 +- Taskfile.yml | 19 +- dbt/macros/date_key.sql | 7 + dbt/macros/scd2_repository_join.sql | 18 + dbt/macros/source_or_empty.sql | 26 ++ dbt/models/marts/_marts__models.yml | 27 +- dbt/models/marts/bridge_issue_labels.sql | 28 +- dbt/models/marts/dim_labels.sql | 29 +- dbt/models/marts/fct_commits.sql | 20 +- .../fct_contributor_activity_monthly.sql | 2 +- dbt/models/marts/fct_daily_downloads.sql | 21 +- dbt/models/marts/fct_issues.sql | 16 +- dbt/models/marts/fct_pull_requests.sql | 18 +- dbt/models/staging/github/_github__models.yml | 12 + .../staging/github/_github__sources.yml | 16 +- .../staging/github/stg_github__commits.sql | 43 +- .../github/stg_github__issue_labels.sql | 21 + .../staging/github/stg_github__issues.sql | 35 +- .../staging/github/stg_github__pr_labels.sql | 19 + .../github/stg_github__pull_requests.sql | 32 +- .../staging/github/stg_github__releases.sql | 17 +- .../github/stg_github__repositories.sql | 53 ++- dbt/models/staging/pypi/_pypi__sources.yml | 2 +- .../staging/pypi/stg_pypi__downloads.sql | 15 +- dbt/seeds/_seeds.yml | 16 + dbt/seeds/projects.csv | 6 + pyproject.toml | 3 +- scripts/.gitkeep | 0 src/repolytics/config.py | 33 +- src/repolytics/ingestion/_http.py | 19 - src/repolytics/ingestion/_meta.py | 23 + src/repolytics/ingestion/github_client.py | 203 --------- src/repolytics/ingestion/github_source.py | 83 ++++ src/repolytics/ingestion/pipeline.py | 43 ++ src/repolytics/ingestion/pypi_client.py | 85 ---- src/repolytics/ingestion/pypi_source.py | 36 ++ src/repolytics/ingestion/watermarks.py | 30 -- src/repolytics/ingestion/writer.py | 69 --- src/repolytics/loading/__init__.py | 0 src/repolytics/loading/raw_loader.py | 54 --- tests/conftest.py | 28 +- tests/fixtures/github_responses/commits.json | 4 +- tests/fixtures/github_responses/pulls.json | 8 - tests/fixtures/pypi_responses/recent.json | 5 - tests/integration/test_pipeline.py | 130 ++++++ tests/integration/test_raw_loader.py | 121 ------ tests/unit/_clients.py | 27 -- tests/unit/test_config.py | 43 +- tests/unit/test_github_client.py | 250 ----------- tests/unit/test_meta.py | 22 + tests/unit/test_pypi_client.py | 105 ----- tests/unit/test_watermarks.py | 33 -- tests/unit/test_writer.py | 84 ---- uv.lock | 398 +++++++++++++++--- 55 files changed, 1065 insertions(+), 1402 deletions(-) create mode 100644 dbt/macros/date_key.sql create mode 100644 dbt/macros/scd2_repository_join.sql create mode 100644 dbt/macros/source_or_empty.sql create mode 100644 dbt/models/staging/github/stg_github__issue_labels.sql create mode 100644 dbt/models/staging/github/stg_github__pr_labels.sql create mode 100644 dbt/seeds/_seeds.yml create mode 100644 dbt/seeds/projects.csv delete mode 100644 scripts/.gitkeep delete mode 100644 src/repolytics/ingestion/_http.py create mode 100644 src/repolytics/ingestion/_meta.py delete mode 100644 src/repolytics/ingestion/github_client.py create mode 100644 src/repolytics/ingestion/github_source.py create mode 100644 src/repolytics/ingestion/pipeline.py delete mode 100644 src/repolytics/ingestion/pypi_client.py create mode 100644 src/repolytics/ingestion/pypi_source.py delete mode 100644 src/repolytics/ingestion/watermarks.py delete mode 100644 src/repolytics/ingestion/writer.py delete mode 100644 src/repolytics/loading/__init__.py delete mode 100644 src/repolytics/loading/raw_loader.py delete mode 100644 tests/fixtures/pypi_responses/recent.json create mode 100644 tests/integration/test_pipeline.py delete mode 100644 tests/integration/test_raw_loader.py delete mode 100644 tests/unit/_clients.py delete mode 100644 tests/unit/test_github_client.py create mode 100644 tests/unit/test_meta.py delete mode 100644 tests/unit/test_pypi_client.py delete mode 100644 tests/unit/test_watermarks.py delete mode 100644 tests/unit/test_writer.py diff --git a/.env.example b/.env.example index 1934254..6303557 100644 --- a/.env.example +++ b/.env.example @@ -1,14 +1,9 @@ # GitHub GITHUB_TOKEN=ghp_xxxxxxxxxxxxxxxxxxxx -GITHUB_TARGET_REPOS=fastapi/fastapi,pydantic/pydantic,pola-rs/polars,duckdb/duckdb,encode/httpx # DuckDB DUCKDB_PATH=data/warehouse/repolytics.duckdb -# Paths -RAW_DATA_PATH=data/raw -WATERMARKS_PATH=data/raw/.watermarks.json - # Airflow AIRFLOW_HOME=orchestration AIRFLOW__CORE__DAGS_FOLDER=orchestration/dags diff --git a/.gitignore b/.gitignore index d1e9704..1b5f2ce 100644 --- a/.gitignore +++ b/.gitignore @@ -21,11 +21,14 @@ venv/ .coverage.* htmlcov/ -# Runtime data (DuckDB warehouse, raw Parquet landing zone) +# Runtime data (DuckDB warehouse) data/ *.duckdb *.duckdb.wal +# dlt (local config/secrets + pipeline working dir) +.dlt/ + # Logs logs/ orchestration/logs/ diff --git a/Taskfile.yml b/Taskfile.yml index e59748b..92d7ade 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -14,6 +14,11 @@ tasks: cmds: - uv run ruff check . + lint:fix: + desc: Run ruff linter (applies fixes) + cmds: + - uv run ruff check --fix . + format: desc: Run ruff formatter (modifies files) cmds: @@ -54,13 +59,19 @@ tasks: - task: dbt:deps - task: dbt:compile - # ----- Loading ----- - load:raw: - desc: Load landed Parquet into the DuckDB raw schema + # ----- dlt ----- + dlt:ingest: + desc: Run the dlt pipeline (GitHub + PyPI) into the DuckDB raw schema cmds: - - uv run python -c "from repolytics.loading.raw_loader import load_all; print(load_all())" + - uv run python -m repolytics.ingestion.pipeline # ----- dbt ----- + dbt:seed: + desc: Load CSV seeds into the warehouse + dir: dbt + cmds: + - uv run --group dbt dbt seed + dbt:deps: desc: Install dbt packages dir: dbt diff --git a/dbt/macros/date_key.sql b/dbt/macros/date_key.sql new file mode 100644 index 0000000..fdaf611 --- /dev/null +++ b/dbt/macros/date_key.sql @@ -0,0 +1,7 @@ +{#- + Surrogate date key (YYYYMMDD integer) from a timestamp/date expression. + Centralizes the format used by every fact join to dim_dates. +-#} +{% macro date_key(ts) -%} + cast(strftime({{ ts }}, '%Y%m%d') as integer) +{%- endmacro %} diff --git a/dbt/macros/scd2_repository_join.sql b/dbt/macros/scd2_repository_join.sql new file mode 100644 index 0000000..574d506 --- /dev/null +++ b/dbt/macros/scd2_repository_join.sql @@ -0,0 +1,18 @@ +{#- + Resolve a fact repository_key against the SCD2 dim_repositories using a + half-open date range (event_date >= valid_from AND < valid_to), so each event + maps to the repository version that was current when it occurred. Centralizes + the SCD2 join contract (half-open bounds; earliest version backdated, current + version valid_to = 9999-12-31) that every fact shares. Select + `.repository_key` after using this. + + repo_expr - SQL expression for the repository full name (owner/name) + date_expr - SQL expression for the event date (cast to date) + alias - alias bound to dim_repositories (default 'r') +-#} +{% macro scd2_repository_join(repo_expr, date_expr, alias='r') -%} +left join {{ ref('dim_repositories') }} {{ alias }} + on {{ repo_expr }} = {{ alias }}.repository_name + and {{ date_expr }} >= {{ alias }}.valid_from + and {{ date_expr }} < {{ alias }}.valid_to +{%- endmacro %} diff --git a/dbt/macros/source_or_empty.sql b/dbt/macros/source_or_empty.sql new file mode 100644 index 0000000..ec0ab09 --- /dev/null +++ b/dbt/macros/source_or_empty.sql @@ -0,0 +1,26 @@ +{#- + Return `select * from ` when the source relation exists, otherwise an + empty result with the given columns (`select cast(null as ) as ... where false`). + + dlt only materializes a normalized child table when some row populated the + underlying list, so a list that is empty across every ingested row leaves the + child table absent. Wrapping the source in this macro keeps the build resilient + to that case. + + `columns` is a mapping of column name -> SQL type, covering the columns the caller reads. +-#} +{% macro source_or_empty(source_name, table_name, columns) -%} + {%- set rel = source(source_name, table_name) -%} + {%- set existing = adapter.get_relation( + database=rel.database, schema=rel.schema, identifier=rel.identifier + ) -%} + {%- if existing is not none -%} + select * from {{ rel }} + {%- else -%} + select + {%- for col, dtype in columns.items() %} + cast(null as {{ dtype }}) as {{ col }}{{ "," if not loop.last }} + {%- endfor %} + where false + {%- endif -%} +{%- endmacro %} diff --git a/dbt/models/marts/_marts__models.yml b/dbt/models/marts/_marts__models.yml index 08cc882..d538b04 100644 --- a/dbt/models/marts/_marts__models.yml +++ b/dbt/models/marts/_marts__models.yml @@ -71,16 +71,6 @@ models: arguments: to: ref('dim_dates') field: date_key - - name: lines_added - data_tests: - - dbt_utils.expression_is_true: - arguments: - expression: ">= 0" - - name: lines_deleted - data_tests: - - dbt_utils.expression_is_true: - arguments: - expression: ">= 0" - name: fct_pull_requests description: One row per pull request. Repository resolved via the SCD2 half-open range on the PR open date. @@ -116,11 +106,6 @@ models: arguments: to: ref('dim_dates') field: date_key - - name: review_count - data_tests: - - dbt_utils.expression_is_true: - arguments: - expression: ">= 0" - name: fct_issues description: One row per issue. Repository resolved via the SCD2 half-open range. @@ -186,8 +171,7 @@ models: - name: fct_daily_downloads description: > - One row per package per day (PyPI 'without_mirrors' downloads). `package` is a - degenerate dimension - no package->repository mapping exists yet. Incremental + One row per package per day (PyPI 'without_mirrors' downloads). Incremental (delete+insert on download_key). columns: - name: download_key @@ -195,6 +179,15 @@ models: data_tests: [not_null, unique] - name: package description: PyPI package name (degenerate dimension). + - name: repository_key + description: > + FK to dim_repositories via the projects (package->repo) seed; nullable when + the package has no mapped/ingested repository. + data_tests: + - relationships: + arguments: + to: ref('dim_repositories') + field: repository_key - name: date_key description: FK to dim_dates (download day, YYYYMMDD). data_tests: diff --git a/dbt/models/marts/bridge_issue_labels.sql b/dbt/models/marts/bridge_issue_labels.sql index e24aded..d4e65b0 100644 --- a/dbt/models/marts/bridge_issue_labels.sql +++ b/dbt/models/marts/bridge_issue_labels.sql @@ -1,27 +1,15 @@ --- Bridge table for the issue<->label many-to-many. Labels land in staging as a JSON --- array per issue; we explode them, recompute the issue surrogate key the same way as --- fct_issues, and resolve each label name to its label_key in dim_labels. +-- Bridge table for the issue<->label many-to-many. Built from the flattened issue +-- label staging model; recomputes the issue surrogate key the same way as fct_issues +-- and resolves each label name to its label_key in dim_labels. -- Grain: one row per (issue, label). with issue_labels as ( - select - repository, - issue_number, - unnest(json_extract(labels, '$[*]')) as label - from {{ ref('stg_github__issues') }} - where labels is not null -), - -exploded as ( - select - {{ dbt_utils.generate_surrogate_key(['repository', 'issue_number']) }} as issue_key, - label ->> '$.name' as label_name - from issue_labels - where (label ->> '$.name') is not null + select * from {{ ref('stg_github__issue_labels') }} ) select distinct - e.issue_key, + {{ dbt_utils.generate_surrogate_key(['il.repository', 'il.issue_number']) }} + as issue_key, dl.label_key -from exploded e -inner join {{ ref('dim_labels') }} dl on e.label_name = dl.label_name +from issue_labels il +inner join {{ ref('dim_labels') }} dl on il.label_name = dl.label_name diff --git a/dbt/models/marts/dim_labels.sql b/dbt/models/marts/dim_labels.sql index 5c7c656..de1e6cf 100644 --- a/dbt/models/marts/dim_labels.sql +++ b/dbt/models/marts/dim_labels.sql @@ -1,28 +1,19 @@ --- Type 1 label dimension: distinct issue/PR labels across all projects. Labels land --- in staging as a JSON array per row, so we explode the arrays and dedupe by name. +-- Type 1 label dimension: distinct issue/PR labels across all projects. Labels come +-- from the flattened dlt child-table staging models, deduped by name. -with labels_raw as ( - select labels - from {{ ref('stg_github__pull_requests') }} - where labels is not null +with labels as ( + select label_name, label_color from {{ ref('stg_github__issue_labels') }} union all - select labels - from {{ ref('stg_github__issues') }} - where labels is not null -), - -exploded as ( - select unnest(json_extract(labels, '$[*]')) as label - from labels_raw + select label_name, label_color from {{ ref('stg_github__pr_labels') }} ), distinct_labels as ( select - label ->> '$.name' as label_name, - max(label ->> '$.color') as label_color - from exploded - where (label ->> '$.name') is not null - group by label ->> '$.name' + label_name, + max(label_color) as label_color + from labels + where label_name is not null + group by label_name ) select diff --git a/dbt/models/marts/fct_commits.sql b/dbt/models/marts/fct_commits.sql index f7775a5..6d630e6 100644 --- a/dbt/models/marts/fct_commits.sql +++ b/dbt/models/marts/fct_commits.sql @@ -2,8 +2,10 @@ -- half-open date range (event_date >= valid_from AND < valid_to) so each commit maps -- to the repository version that was current when it landed; to dim_contributors on -- author login; and to dim_dates via an inline YYYYMMDD key. --- Incremental (delete+insert on commit_key): each run only processes commits at or --- after the latest committed_at already loaded; the unique key keeps it idempotent. +-- Incremental (delete+insert on commit_key): each run only processes rows loaded +-- since the last run, keyed on the monotonic dlt `_loaded_at` (the git author date is +-- not monotonic, so a rebased/old-authored commit pushed today would be skipped). +-- The unique key keeps it idempotent. {{ config( @@ -17,7 +19,7 @@ with commits as ( select * from {{ ref('stg_github__commits') }} {% if is_incremental() %} - where committed_at >= (select max(committed_at) from {{ this }}) + where _loaded_at >= (select max(_loaded_at) from {{ this }}) {% endif %} ) @@ -25,15 +27,11 @@ select {{ dbt_utils.generate_surrogate_key(['c.repository', 'c.commit_sha']) }} as commit_key, r.repository_key, co.contributor_key, - cast(strftime(c.committed_at, '%Y%m%d') as integer) as date_key, + {{ date_key('c.committed_at') }} as date_key, c.commit_sha as commit_hash, - c.additions as lines_added, - c.deletions as lines_deleted, - c.committed_at + c.committed_at, + c._loaded_at from commits c -left join {{ ref('dim_repositories') }} r - on c.repository = r.repository_name - and c.committed_at::date >= r.valid_from - and c.committed_at::date < r.valid_to +{{ scd2_repository_join('c.repository', 'c.committed_at::date') }} left join {{ ref('dim_contributors') }} co on c.author_login = co.username diff --git a/dbt/models/marts/fct_contributor_activity_monthly.sql b/dbt/models/marts/fct_contributor_activity_monthly.sql index ecd1f6e..0196e08 100644 --- a/dbt/models/marts/fct_contributor_activity_monthly.sql +++ b/dbt/models/marts/fct_contributor_activity_monthly.sql @@ -21,7 +21,7 @@ select c.contributor_key, m.username, m.event_month, - cast(strftime(m.event_month, '%Y%m%d') as integer) as month_date_key, + {{ date_key('m.event_month') }} as month_date_key, m.commits, m.prs_opened, m.prs_merged, diff --git a/dbt/models/marts/fct_daily_downloads.sql b/dbt/models/marts/fct_daily_downloads.sql index 91d7799..8e5a050 100644 --- a/dbt/models/marts/fct_daily_downloads.sql +++ b/dbt/models/marts/fct_daily_downloads.sql @@ -1,4 +1,6 @@ --- PyPI daily download fact: one row per package per day. +-- PyPI daily download fact: one row per package per day. `repository_key` is resolved +-- through the `projects` seed (package -> repo) and the SCD2 dim_repositories half-open +-- range; it is nullable for packages with no mapped/ingested repo. -- Filtered to the 'without_mirrors' overall time-series category to avoid double -- counting 'with_mirrors' and to exclude the recent-endpoint 'last_*' aggregates. -- Incremental (delete+insert on download_key): only processes days at or after the @@ -18,14 +20,17 @@ with downloads as ( where category = 'without_mirrors' {% if is_incremental() %} -- date_key is the only date column on the target table; compare the day's key. - and cast(strftime(download_date, '%Y%m%d') as integer) - >= (select max(date_key) from {{ this }}) + and {{ date_key('download_date') }} >= (select max(date_key) from {{ this }}) {% endif %} ) select - {{ dbt_utils.generate_surrogate_key(['package', 'download_date']) }} as download_key, - package, - cast(strftime(download_date, '%Y%m%d') as integer) as date_key, - download_count -from downloads + {{ dbt_utils.generate_surrogate_key(['d.package', 'd.download_date']) }} + as download_key, + d.package, + r.repository_key, + {{ date_key('d.download_date') }} as date_key, + d.download_count +from downloads d +left join {{ ref('projects') }} p on d.package = p.package +{{ scd2_repository_join('p.repo', 'd.download_date') }} diff --git a/dbt/models/marts/fct_issues.sql b/dbt/models/marts/fct_issues.sql index 9a43bf7..daebf34 100644 --- a/dbt/models/marts/fct_issues.sql +++ b/dbt/models/marts/fct_issues.sql @@ -10,15 +10,15 @@ select {{ dbt_utils.generate_surrogate_key(['i.repository', 'i.issue_number']) }} as issue_key, r.repository_key, co.contributor_key as author_key, - cast(strftime(i.created_at, '%Y%m%d') as integer) as opened_date_key, - cast(strftime(i.closed_at, '%Y%m%d') as integer) as closed_date_key, - i.is_closed, - i.time_to_close_hours, + {{ date_key('i.created_at') }} as opened_date_key, + {{ date_key('i.closed_at') }} as closed_date_key, + i.state = 'closed' as is_closed, + case + when i.closed_at is not null + then datediff('hour', i.created_at, i.closed_at) + end as time_to_close_hours, i.comment_count from issues i -left join {{ ref('dim_repositories') }} r - on i.repository = r.repository_name - and i.created_at::date >= r.valid_from - and i.created_at::date < r.valid_to +{{ scd2_repository_join('i.repository', 'i.created_at::date') }} left join {{ ref('dim_contributors') }} co on i.author_login = co.username diff --git a/dbt/models/marts/fct_pull_requests.sql b/dbt/models/marts/fct_pull_requests.sql index fa8b464..88b75e8 100644 --- a/dbt/models/marts/fct_pull_requests.sql +++ b/dbt/models/marts/fct_pull_requests.sql @@ -10,18 +10,14 @@ select {{ dbt_utils.generate_surrogate_key(['p.repository', 'p.pr_number']) }} as pr_key, r.repository_key, co.contributor_key as author_key, - cast(strftime(p.created_at, '%Y%m%d') as integer) as opened_date_key, - cast(strftime(p.merged_at, '%Y%m%d') as integer) as merged_date_key, + {{ date_key('p.created_at') }} as opened_date_key, + {{ date_key('p.merged_at') }} as merged_date_key, p.merged_at is not null as is_merged, - p.time_to_merge_hours, - p.review_comments as review_count, - p.comment_count, - p.additions, - p.deletions + case + when p.merged_at is not null + then datediff('hour', p.created_at, p.merged_at) + end as time_to_merge_hours from pull_requests p -left join {{ ref('dim_repositories') }} r - on p.repository = r.repository_name - and p.created_at::date >= r.valid_from - and p.created_at::date < r.valid_to +{{ scd2_repository_join('p.repository', 'p.created_at::date') }} left join {{ ref('dim_contributors') }} co on p.author_login = co.username diff --git a/dbt/models/staging/github/_github__models.yml b/dbt/models/staging/github/_github__models.yml index 14cc18a..98db201 100644 --- a/dbt/models/staging/github/_github__models.yml +++ b/dbt/models/staging/github/_github__models.yml @@ -69,3 +69,15 @@ models: data_tests: [not_null, unique] - name: repository data_tests: [not_null] + + - name: stg_github__issue_labels + description: One row per (issue, label), flattened from the dlt child table. + columns: + - name: label_name + data_tests: [not_null] + + - name: stg_github__pr_labels + description: One row per (pull request, label), flattened from the dlt child table. + columns: + - name: label_name + data_tests: [not_null] diff --git a/dbt/models/staging/github/_github__sources.yml b/dbt/models/staging/github/_github__sources.yml index c17b8df..f795764 100644 --- a/dbt/models/staging/github/_github__sources.yml +++ b/dbt/models/staging/github/_github__sources.yml @@ -2,7 +2,7 @@ version: 2 sources: - name: github - description: Raw GitHub API responses, landed as a JSON `data` blob per record. + description: GitHub API data loaded and normalized by dlt into typed tables. schema: raw loaded_at_field: _loaded_at freshness: @@ -10,12 +10,24 @@ sources: error_after: {count: 24, period: hour} tables: - name: repositories - description: One row per repository snapshot. + description: One row per repository (current snapshot). + - name: repositories__topics + description: dlt child table; one row per repository topic. + freshness: null # child table has no _loaded_at - name: commits description: One row per commit; `_repo` carries the source repository. + - name: commits__parents + description: dlt child table; one row per commit parent (merge detection). + freshness: null - name: pull_requests description: One row per pull request; `_repo` carries the source repository. + - name: pull_requests__labels + description: dlt child table; one row per pull-request label. + freshness: null - name: issues description: One row per issue (includes PRs upstream; staging filters them). + - name: issues__labels + description: dlt child table; one row per issue label. + freshness: null - name: releases description: One row per release; `_repo` carries the source repository. diff --git a/dbt/models/staging/github/stg_github__commits.sql b/dbt/models/staging/github/stg_github__commits.sql index 34d443c..45557c3 100644 --- a/dbt/models/staging/github/stg_github__commits.sql +++ b/dbt/models/staging/github/stg_github__commits.sql @@ -1,19 +1,30 @@ -with source as ( - select data::json as d, _repo, _loaded_at - from {{ source('github', 'commits') }} +-- Structural cleaning over dlt's normalized `commits` table. Merge commits (more +-- than one parent) are dropped using a count from the dlt `commits__parents` child table. + +with commits as ( + select * from {{ source('github', 'commits') }} +), + +parents_src as ( + {{ source_or_empty('github', 'commits__parents', {'_dlt_parent_id': 'varchar'}) }} +), + +parent_counts as ( + select _dlt_parent_id, count(*) as parent_count + from parents_src + group by _dlt_parent_id ) select - d ->> '$.sha' as commit_sha, - _repo as repository, - d ->> '$.author.login' as author_login, - d ->> '$.commit.author.name' as author_name, - d ->> '$.commit.author.email' as author_email, - (d ->> '$.commit.author.date')::timestamp as committed_at, - (d ->> '$.stats.additions')::bigint as additions, - (d ->> '$.stats.deletions')::bigint as deletions, - d ->> '$.commit.message' as message, - _loaded_at -from source --- Drop merge commits (more than one parent); rows without `parents` are kept. -where coalesce(json_array_length(d -> '$.parents'), 0) <= 1 + c.sha as commit_sha, + c._repo as repository, + c.author__login as author_login, + c.commit__author__name as author_name, + c.commit__author__email as author_email, + c.commit__author__date as committed_at, + c.commit__message as message, + c._loaded_at +from commits c +left join parent_counts p on c._dlt_id = p._dlt_parent_id +-- Keep non-merge commits (<= 1 parent); commits with no parents are kept too. +where coalesce(p.parent_count, 0) <= 1 diff --git a/dbt/models/staging/github/stg_github__issue_labels.sql b/dbt/models/staging/github/stg_github__issue_labels.sql new file mode 100644 index 0000000..9bc94c3 --- /dev/null +++ b/dbt/models/staging/github/stg_github__issue_labels.sql @@ -0,0 +1,21 @@ +-- Flattens dlt's `issues__labels` child table back to its parent issue, exposing a +-- repository + issue_number grain. Labels on PR-shaped issues are excluded to +-- match the issues staging filter. + +with labels as ( + {{ source_or_empty('github', 'issues__labels', + {'name': 'varchar', 'color': 'varchar', '_dlt_parent_id': 'varchar'}) }} +), + +issues as ( + select * from {{ source('github', 'issues') }} + where pull_request__url is null +) + +select + i._repo as repository, + i.number as issue_number, + l.name as label_name, + l.color as label_color +from labels l +inner join issues i on l._dlt_parent_id = i._dlt_id diff --git a/dbt/models/staging/github/stg_github__issues.sql b/dbt/models/staging/github/stg_github__issues.sql index 2e10e86..4b25f2f 100644 --- a/dbt/models/staging/github/stg_github__issues.sql +++ b/dbt/models/staging/github/stg_github__issues.sql @@ -1,26 +1,19 @@ -with source as ( - select data::json as d, _repo, _loaded_at - from {{ source('github', 'issues') }} +-- Structural cleaning over dlt's normalized `issues` table. The issues endpoint +-- returns PRs too; drop them (real issues have no `pull_request`). Labels live in +-- the `issues__labels` child table (see stg_github__issue_labels). + +with issues as ( + select * from {{ source('github', 'issues') }} ) select _repo as repository, - (d ->> '$.number')::bigint as issue_number, - d ->> '$.user.login' as author_login, - d ->> '$.state' as state, - (d ->> '$.state') = 'closed' as is_closed, - (d ->> '$.created_at')::timestamp as created_at, - (d ->> '$.closed_at')::timestamp as closed_at, - case - when (d ->> '$.closed_at') is not null then datediff( - 'hour', - (d ->> '$.created_at')::timestamp, - (d ->> '$.closed_at')::timestamp - ) - end as time_to_close_hours, - (d ->> '$.comments')::bigint as comment_count, - d -> '$.labels' as labels, + number as issue_number, + user__login as author_login, + state, + created_at, + closed_at, + comments as comment_count, _loaded_at -from source --- The issues endpoint returns PRs too; drop them (real issues have no `pull_request`). -where (d -> '$.pull_request') is null +from issues +where pull_request__url is null diff --git a/dbt/models/staging/github/stg_github__pr_labels.sql b/dbt/models/staging/github/stg_github__pr_labels.sql new file mode 100644 index 0000000..6ead715 --- /dev/null +++ b/dbt/models/staging/github/stg_github__pr_labels.sql @@ -0,0 +1,19 @@ +-- Flattens dlt's `pull_requests__labels` child table back to its parent PR, +-- exposing a repository + pr_number grain. + +with labels as ( + {{ source_or_empty('github', 'pull_requests__labels', + {'name': 'varchar', 'color': 'varchar', '_dlt_parent_id': 'varchar'}) }} +), + +pull_requests as ( + select * from {{ source('github', 'pull_requests') }} +) + +select + p._repo as repository, + p.number as pr_number, + l.name as label_name, + l.color as label_color +from labels l +inner join pull_requests p on l._dlt_parent_id = p._dlt_id diff --git a/dbt/models/staging/github/stg_github__pull_requests.sql b/dbt/models/staging/github/stg_github__pull_requests.sql index 57757ba..5e426e6 100644 --- a/dbt/models/staging/github/stg_github__pull_requests.sql +++ b/dbt/models/staging/github/stg_github__pull_requests.sql @@ -1,26 +1,16 @@ -with source as ( - select data::json as d, _repo, _loaded_at - from {{ source('github', 'pull_requests') }} +-- Structural cleaning over dlt's normalized `pull_requests` table. Labels live in +-- the `pull_requests__labels` child table (see stg_github__pr_labels). + +with pull_requests as ( + select * from {{ source('github', 'pull_requests') }} ) select _repo as repository, - (d ->> '$.number')::bigint as pr_number, - d ->> '$.user.login' as author_login, - d ->> '$.state' as state, - (d ->> '$.created_at')::timestamp as created_at, - (d ->> '$.merged_at')::timestamp as merged_at, - (d ->> '$.additions')::bigint as additions, - (d ->> '$.deletions')::bigint as deletions, - (d ->> '$.review_comments')::bigint as review_comments, - (d ->> '$.comments')::bigint as comment_count, - case - when (d ->> '$.merged_at') is not null then datediff( - 'hour', - (d ->> '$.created_at')::timestamp, - (d ->> '$.merged_at')::timestamp - ) - end as time_to_merge_hours, - d -> '$.labels' as labels, + number as pr_number, + user__login as author_login, + state, + created_at, + merged_at, _loaded_at -from source +from pull_requests diff --git a/dbt/models/staging/github/stg_github__releases.sql b/dbt/models/staging/github/stg_github__releases.sql index ceb118f..678d2a4 100644 --- a/dbt/models/staging/github/stg_github__releases.sql +++ b/dbt/models/staging/github/stg_github__releases.sql @@ -1,13 +1,14 @@ -with source as ( - select data::json as d, _repo, _loaded_at - from {{ source('github', 'releases') }} +-- Structural cleaning over dlt's normalized `releases` table. + +with releases as ( + select * from {{ source('github', 'releases') }} ) select - (d ->> '$.id')::bigint as release_id, + id as release_id, _repo as repository, - d ->> '$.tag_name' as tag_name, - d ->> '$.name' as name, - (d ->> '$.published_at')::timestamp as published_at, + tag_name, + name, + published_at, _loaded_at -from source +from releases diff --git a/dbt/models/staging/github/stg_github__repositories.sql b/dbt/models/staging/github/stg_github__repositories.sql index ecd7011..656769a 100644 --- a/dbt/models/staging/github/stg_github__repositories.sql +++ b/dbt/models/staging/github/stg_github__repositories.sql @@ -1,21 +1,38 @@ -with source as ( - select data::json as d, _loaded_at - from {{ source('github', 'repositories') }} +-- Structural cleaning over dlt's normalized `repositories` table. Topics are +-- re-aggregated from the dlt child table into a sorted comma string so the SCD2 +-- snapshot's check_cols see a stable scalar. + +with repositories as ( + select * from {{ source('github', 'repositories') }} +), + +topics_src as ( + {{ source_or_empty('github', 'repositories__topics', + {'value': 'varchar', '_dlt_parent_id': 'varchar'}) }} +), + +topics as ( + select + _dlt_parent_id, + string_agg(value, ',' order by value) as topics + from topics_src + group by _dlt_parent_id ) select - (d ->> '$.id')::bigint as repository_id, - d ->> '$.full_name' as repository_name, - d ->> '$.name' as name, - d ->> '$.owner.login' as owner_login, - d ->> '$.description' as description, - (d ->> '$.stargazers_count')::bigint as stars, - (d ->> '$.forks_count')::bigint as forks, - (d ->> '$.open_issues_count')::bigint as open_issues, - d ->> '$.language' as language, - d ->> '$.license.spdx_id' as license_spdx, - d -> '$.topics' as topics, - (d ->> '$.created_at')::timestamp as created_at, - (d ->> '$.updated_at')::timestamp as updated_at, - _loaded_at -from source + r.id as repository_id, + r.full_name as repository_name, + r.name, + r.owner__login as owner_login, + r.description, + r.stargazers_count as stars, + r.forks_count as forks, + r.open_issues_count as open_issues, + r.language, + r.license__spdx_id as license_spdx, + t.topics, + r.created_at, + r.updated_at, + r._loaded_at +from repositories r +left join topics t on r._dlt_id = t._dlt_parent_id diff --git a/dbt/models/staging/pypi/_pypi__sources.yml b/dbt/models/staging/pypi/_pypi__sources.yml index 189caf4..73ed8d0 100644 --- a/dbt/models/staging/pypi/_pypi__sources.yml +++ b/dbt/models/staging/pypi/_pypi__sources.yml @@ -2,7 +2,7 @@ version: 2 sources: - name: pypi - description: Raw PyPI Stats responses; one row per package/day/category. + description: PyPI Stats data loaded by dlt; one row per package/day/category. schema: raw loaded_at_field: _loaded_at freshness: diff --git a/dbt/models/staging/pypi/stg_pypi__downloads.sql b/dbt/models/staging/pypi/stg_pypi__downloads.sql index 979d20b..058187c 100644 --- a/dbt/models/staging/pypi/stg_pypi__downloads.sql +++ b/dbt/models/staging/pypi/stg_pypi__downloads.sql @@ -1,12 +1,13 @@ -with source as ( - select data::json as d, _package, _loaded_at - from {{ source('pypi', 'downloads') }} +-- Structural cleaning over dlt's normalized `downloads` table. + +with downloads as ( + select * from {{ source('pypi', 'downloads') }} ) select _package as package, - d ->> '$.category' as category, - (d ->> '$.date')::date as download_date, - (d ->> '$.downloads')::bigint as download_count, + category, + date::date as download_date, + downloads as download_count, _loaded_at -from source +from downloads diff --git a/dbt/seeds/_seeds.yml b/dbt/seeds/_seeds.yml new file mode 100644 index 0000000..1b73027 --- /dev/null +++ b/dbt/seeds/_seeds.yml @@ -0,0 +1,16 @@ +version: 2 + +seeds: + - name: projects + description: > + Single source of truth linking each tracked GitHub repo to its PyPI + package. Loaded by dbt to give the downloads fact a repository foreign + key, and read by the ingestion pipeline to drive which repos/packages + are extracted. + columns: + - name: repo + description: GitHub repository in owner/name form. + data_tests: [not_null, unique] + - name: package + description: PyPI package name (blank for repos not published to PyPI). + data_tests: [unique] diff --git a/dbt/seeds/projects.csv b/dbt/seeds/projects.csv new file mode 100644 index 0000000..73991f0 --- /dev/null +++ b/dbt/seeds/projects.csv @@ -0,0 +1,6 @@ +repo,package +fastapi/fastapi,fastapi +pydantic/pydantic,pydantic +pola-rs/polars,polars +duckdb/duckdb,duckdb +encode/httpx,httpx diff --git a/pyproject.toml b/pyproject.toml index b491a85..c8f9ea4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,9 +5,8 @@ description = "End-to-end ELT pipeline for open source project analytics (GitHub readme = "README.md" requires-python = ">=3.13" dependencies = [ + "dlt[duckdb]>=1.28.0", "duckdb>=1.5.3", - "httpx>=0.28.1", - "polars>=1.41.2", "pydantic-settings>=2.14.1", ] diff --git a/scripts/.gitkeep b/scripts/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/src/repolytics/config.py b/src/repolytics/config.py index b308548..2fe2a20 100644 --- a/src/repolytics/config.py +++ b/src/repolytics/config.py @@ -3,7 +3,8 @@ See `.env.example` for the full set of variables. """ -from functools import lru_cache +import csv +from functools import cached_property, lru_cache from pathlib import Path from pydantic import SecretStr @@ -21,24 +22,34 @@ class Settings(BaseSettings): # GitHub github_token: SecretStr - github_target_repos: str = "" # DuckDB duckdb_path: Path = Path("data/warehouse/repolytics.duckdb") - # Paths - raw_data_path: Path = Path("data/raw") - watermarks_path: Path = Path("data/raw/.watermarks.json") + # Project list (repo <-> package) - also loaded by dbt as the `projects` seed. + projects_file: Path = Path("dbt/seeds/projects.csv") + + @cached_property + def projects(self) -> list[dict[str, str]]: + """Projects to ingest, each a `{repo, package}` row from `projects_file`. + + `package` may be blank for repos not published to PyPI. Returns an empty + list when the file is absent. + """ + if not self.projects_file.exists(): + return [] + with self.projects_file.open(newline="", encoding="utf-8") as file: + return list(csv.DictReader(file)) @property def target_repos(self) -> list[str]: - """Target repositories as a clean `owner/name` list. + """Target repositories as a clean `owner/name` list.""" + return [p["repo"] for p in self.projects if p.get("repo")] - Parses the comma-separated `GITHUB_TARGET_REPOS` value. - """ - return [ - repo.strip() for repo in self.github_target_repos.split(",") if repo.strip() - ] + @property + def packages(self) -> list[str]: + """Target PyPI packages as a clean list (projects without a package skipped).""" + return [p["package"] for p in self.projects if p.get("package")] @lru_cache diff --git a/src/repolytics/ingestion/_http.py b/src/repolytics/ingestion/_http.py deleted file mode 100644 index 0245c00..0000000 --- a/src/repolytics/ingestion/_http.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Shared HTTP helpers for the ingestion API clients.""" - -import random - -import httpx - -# HTTP statuses worth retrying. -RETRYABLE_STATUSES = frozenset({429, 500, 502, 503}) - - -def retry_delay(response: httpx.Response, attempt: int, backoff_base: float) -> float: - """Seconds to wait before a retry: `Retry-After` if present, else backoff.""" - retry_after = response.headers.get("Retry-After") - if retry_after is not None: - try: - return float(retry_after) - except ValueError: - pass - return backoff_base * 2**attempt + random.uniform(0, backoff_base) diff --git a/src/repolytics/ingestion/_meta.py b/src/repolytics/ingestion/_meta.py new file mode 100644 index 0000000..1f5aca7 --- /dev/null +++ b/src/repolytics/ingestion/_meta.py @@ -0,0 +1,23 @@ +"""Shared extraction-metadata stamping for dlt resources. + +dlt does not attribute sub-resource rows (commits/issues to their repo) or add a +per-row load timestamp by default, so each resource maps its records through +`stamp(...)` to inject `_loaded_at` plus constant `_`-prefixed provenance columns +(`_repo` for GitHub sub-resources, `_package` for PyPI). +""" + +from collections.abc import Callable +from datetime import UTC, datetime + + +def stamp(**metadata: str) -> Callable[[dict], dict]: + """Build a dlt `add_map` function that stamps `_loaded_at` + `metadata`. + + Returns a callable applied to each record; it adds a UTC `_loaded_at` and one + constant column per `metadata` key (e.g. `_repo`/`_package`). + """ + + def _apply(record: dict) -> dict: + return {**record, "_loaded_at": datetime.now(UTC), **metadata} + + return _apply diff --git a/src/repolytics/ingestion/github_client.py b/src/repolytics/ingestion/github_client.py deleted file mode 100644 index e9956bd..0000000 --- a/src/repolytics/ingestion/github_client.py +++ /dev/null @@ -1,203 +0,0 @@ -"""GitHub REST API client. - -Methods return raw JSON exactly as received. Handles authentication, -pagination, rate limiting, and retries with exponential backoff. -""" - -import time -from collections.abc import Callable -from datetime import UTC, datetime - -import httpx - -from repolytics.config import get_settings -from repolytics.ingestion._http import RETRYABLE_STATUSES, retry_delay - - -def _to_iso(dt: datetime) -> str: - """Render a `datetime` as an ISO 8601 UTC string (e.g. `2024-01-01T00:00:00Z`).""" - if dt.tzinfo is None: - dt = dt.replace(tzinfo=UTC) - return dt.astimezone(UTC).isoformat().replace("+00:00", "Z") - - -class GitHubClient: - """Synchronous client for the GitHub REST API. - - Wraps `httpx.Client`. Construct with an explicit token, or use - `GitHubClient.from_settings()` to read it from the environment. Usable as a - context manager so the underlying connection pool is closed on exit. - """ - - def __init__( - self, - token: str, - *, - base_url: str = "https://api.github.com", - per_page: int = 100, - max_retries: int = 3, - backoff_base: float = 1.0, - rate_limit_threshold: int = 1, - timeout: httpx.Timeout | None = None, - transport: httpx.BaseTransport | None = None, - sleep: Callable[[float], None] = time.sleep, - ) -> None: - self._per_page = per_page - self._max_retries = max_retries - self._backoff_base = backoff_base - self._rate_limit_threshold = rate_limit_threshold - self._sleep = sleep - self._client = httpx.Client( - base_url=base_url, - headers={ - "Authorization": f"Bearer {token}", - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2026-03-10", - }, - timeout=timeout - or httpx.Timeout(connect=30.0, read=60.0, write=60.0, pool=60.0), - transport=transport, - ) - - @classmethod - def from_settings(cls, **kwargs: object) -> "GitHubClient": - """Build a client using the token from `repolytics.config.get_settings()`.""" - token = get_settings().github_token.get_secret_value() - return cls(token, **kwargs) - - # --- Lifecycle --- - - def close(self) -> None: - """Close the underlying HTTP connection pool.""" - self._client.close() - - def __enter__(self) -> "GitHubClient": - return self - - def __exit__(self, *exc: object) -> None: - self.close() - - # --- Endpoints --- - - def get_repository(self, owner: str, repo: str) -> dict: - """`GET /repos/{owner}/{repo}` (repository metadata).""" - return self._request("GET", f"/repos/{owner}/{repo}").json() - - def get_commits( - self, owner: str, repo: str, since: datetime | None = None - ) -> list[dict]: - """`GET /repos/{owner}/{repo}/commits`, optionally since a timestamp.""" - params: dict[str, object] = {"per_page": self._per_page} - if since is not None: - params["since"] = _to_iso(since) - return self._paginate(f"/repos/{owner}/{repo}/commits", params) - - def get_issues( - self, owner: str, repo: str, since: datetime | None = None - ) -> list[dict]: - """`GET /repos/{owner}/{repo}/issues?state=all` (includes PRs upstream).""" - params: dict[str, object] = {"state": "all", "per_page": self._per_page} - if since is not None: - params["since"] = _to_iso(since) - return self._paginate(f"/repos/{owner}/{repo}/issues", params) - - def get_pull_requests( - self, owner: str, repo: str, since: datetime | None = None - ) -> list[dict]: - """`GET /repos/{owner}/{repo}/pulls?state=all`. - - The pulls endpoint has no `since` parameter, so for incremental pulls - we sort by `updated` descending and stop paginating once a PR predates - `since`. - """ - params: dict[str, object] = {"state": "all", "per_page": self._per_page} - if since is None: - return self._paginate(f"/repos/{owner}/{repo}/pulls", params) - params["sort"] = "updated" - params["direction"] = "desc" - return self._paginate( - f"/repos/{owner}/{repo}/pulls", params, stop=self._stop_before(since) - ) - - def get_releases(self, owner: str, repo: str) -> list[dict]: - """`GET /repos/{owner}/{repo}/releases`.""" - params: dict[str, object] = {"per_page": self._per_page} - return self._paginate(f"/repos/{owner}/{repo}/releases", params) - - # --- Internals --- - - def _paginate( - self, - url: str, - params: dict[str, object] | None, - *, - stop: Callable[[list[dict]], tuple[list[dict], bool]] | None = None, - ) -> list[dict]: - """Follow `Link: rel="next"` pages, collecting all items. - - `stop` filters each page and signals early termination - (used for the pulls `since` walk). - """ - results: list[dict] = [] - next_url: str | None = url - next_params = params - while next_url is not None: - response = self._request("GET", next_url, params=next_params) - page: list[dict] = response.json() - if stop is not None: - kept, done = stop(page) - results.extend(kept) - if done: - break - else: - results.extend(page) - link = response.links.get("next") - next_url = link["url"] if link else None - next_params = None # the next URL already carries the query string - return results - - def _request( - self, method: str, url: str, params: dict[str, object] | None = None - ) -> httpx.Response: - """Issue one request, retrying transient failures and guarding the limit.""" - response = self._client.request(method, url, params=params) - for attempt in range(self._max_retries): - if response.status_code not in RETRYABLE_STATUSES: - break - self._sleep(retry_delay(response, attempt, self._backoff_base)) - response = self._client.request(method, url, params=params) - response.raise_for_status() - self._guard_rate_limit(response) - return response - - def _guard_rate_limit(self, response: httpx.Response) -> None: - """Sleep until the rate-limit window resets when remaining calls run low.""" - remaining = response.headers.get("X-RateLimit-Remaining") - reset = response.headers.get("X-RateLimit-Reset") - if remaining is None or reset is None: - return - if int(remaining) <= self._rate_limit_threshold: - delay = float(reset) - time.time() - if delay > 0: - self._sleep(delay) - - @staticmethod - def _stop_before( - since: datetime, - ) -> Callable[[list[dict]], tuple[list[dict], bool]]: - """Build a page filter that keeps PRs updated at/after `since`. - - Assumes pages are sorted by `updated_at` descending, so the first PR - older than `since` ends the walk. - """ - cutoff = since if since.tzinfo else since.replace(tzinfo=UTC) - - def stop(page: list[dict]) -> tuple[list[dict], bool]: - kept: list[dict] = [] - for item in page: - if datetime.fromisoformat(item["updated_at"]) < cutoff: - return kept, True - kept.append(item) - return kept, False - - return stop diff --git a/src/repolytics/ingestion/github_source.py b/src/repolytics/ingestion/github_source.py new file mode 100644 index 0000000..5a16b79 --- /dev/null +++ b/src/repolytics/ingestion/github_source.py @@ -0,0 +1,83 @@ +"""dlt source for the GitHub REST API. + +One resource per endpoint (repositories, commits, issues, pull requests, +releases), fetched for each target repo. +""" + +from collections.abc import Iterator + +import dlt +from dlt.sources.helpers.rest_client import RESTClient +from dlt.sources.helpers.rest_client.auth import BearerTokenAuth + +from repolytics.ingestion._meta import stamp + +BASE_URL = "https://api.github.com" +API_VERSION = "2026-03-10" +PER_PAGE = 100 + + +@dlt.source(name="github") +def github_source(repos: list[str], token: str) -> list: + """dlt source exposing the GitHub endpoints for each `owner/name` in `repos`.""" + client = RESTClient( + base_url=BASE_URL, + auth=BearerTokenAuth(token), + headers={ + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": API_VERSION, + }, + ) + + def _paginate(path: str, repo: str, params: dict | None = None) -> Iterator[dict]: + apply = stamp(_repo=repo) + query = {"per_page": PER_PAGE, **(params or {})} + for page in client.paginate(path, params=query): + yield from (apply(item) for item in page) + + @dlt.resource(name="repositories", write_disposition="merge", primary_key="id") + def repositories() -> Iterator[dict]: + for repo in repos: + owner, name = repo.split("/") + response = client.get(f"/repos/{owner}/{name}") + response.raise_for_status() + yield stamp(_repo=repo)(response.json()) + + @dlt.resource(name="commits", write_disposition="merge", primary_key="sha") + def commits() -> Iterator[dict]: + for repo in repos: + owner, name = repo.split("/") + yield from _paginate(f"/repos/{owner}/{name}/commits", repo) + + @dlt.resource( + name="issues", + write_disposition="merge", + primary_key=["_repo", "number"], # issue number is unique per repo + # Force the flattened PR marker column to exist even when a load has no + # PR-shaped issues, so the staging PR filter never references a missing column. + columns={"pull_request__url": {"data_type": "text", "nullable": True}}, + ) + def issues() -> Iterator[dict]: + for repo in repos: + owner, name = repo.split("/") + yield from _paginate( + f"/repos/{owner}/{name}/issues", repo, {"state": "all"} + ) + + @dlt.resource( + name="pull_requests", + write_disposition="merge", + primary_key=["_repo", "number"], # PR number is unique per repo + ) + def pull_requests() -> Iterator[dict]: + for repo in repos: + owner, name = repo.split("/") + yield from _paginate(f"/repos/{owner}/{name}/pulls", repo, {"state": "all"}) + + @dlt.resource(name="releases", write_disposition="merge", primary_key="id") + def releases() -> Iterator[dict]: + for repo in repos: + owner, name = repo.split("/") + yield from _paginate(f"/repos/{owner}/{name}/releases", repo) + + return [repositories, commits, issues, pull_requests, releases] diff --git a/src/repolytics/ingestion/pipeline.py b/src/repolytics/ingestion/pipeline.py new file mode 100644 index 0000000..7bcd718 --- /dev/null +++ b/src/repolytics/ingestion/pipeline.py @@ -0,0 +1,43 @@ +"""dlt pipeline wiring GitHub + PyPI ingestion into the DuckDB `raw` dataset.""" + +import dlt + +from repolytics.config import Settings, get_settings +from repolytics.ingestion.github_source import github_source +from repolytics.ingestion.pypi_source import pypi_source + + +def build_pipeline(settings: Settings) -> dlt.Pipeline: + """Create the DuckDB-backed dlt pipeline targeting the `raw` dataset.""" + return dlt.pipeline( + pipeline_name="repolytics", + destination=dlt.destinations.duckdb(str(settings.duckdb_path)), + dataset_name="raw", + ) + + +def run(settings: Settings | None = None) -> None: + """Run GitHub + PyPI ingestion into the configured DuckDB warehouse.""" + settings = settings or get_settings() + + repos = settings.target_repos + packages = settings.packages + if not repos and not packages: + raise RuntimeError(f"No projects to ingest - check {settings.projects_file}") + + settings.duckdb_path.parent.mkdir(parents=True, exist_ok=True) + pipeline = build_pipeline(settings) + + # Run both sources in a single load so GitHub + PyPI land atomically. + sources = [] + if repos: + sources.append(github_source(repos, settings.github_token.get_secret_value())) + if packages: + sources.append(pypi_source(packages)) + + info = pipeline.run(sources) + print(info) + + +if __name__ == "__main__": + run() diff --git a/src/repolytics/ingestion/pypi_client.py b/src/repolytics/ingestion/pypi_client.py deleted file mode 100644 index c65d59d..0000000 --- a/src/repolytics/ingestion/pypi_client.py +++ /dev/null @@ -1,85 +0,0 @@ -"""PyPI Stats API client. - -Methods return raw JSON exactly as received. Handles retries with exponential -backoff and keeps a courtesy delay between requests to stay within PyPI's -informal rate limits. -""" - -import time -from collections.abc import Callable - -import httpx - -from repolytics.ingestion._http import RETRYABLE_STATUSES, retry_delay - - -class PyPIClient: - """Synchronous client for the PyPI Stats API. - - Wraps `httpx.Client`. Usable as a context manager so the underlying - connection pool is closed on exit. - """ - - def __init__( - self, - *, - base_url: str = "https://pypistats.org/api", - min_interval: float = 1.0, - max_retries: int = 3, - backoff_base: float = 1.0, - timeout: httpx.Timeout | None = None, - transport: httpx.BaseTransport | None = None, - sleep: Callable[[float], None] = time.sleep, - ) -> None: - self._min_interval = min_interval - self._max_retries = max_retries - self._backoff_base = backoff_base - self._sleep = sleep - self._client = httpx.Client( - base_url=base_url, - headers={"Accept": "application/json"}, - timeout=timeout - or httpx.Timeout(connect=30.0, read=60.0, write=60.0, pool=60.0), - transport=transport, - ) - - # --- Lifecycle --- - - def close(self) -> None: - """Close the underlying HTTP connection pool.""" - self._client.close() - - def __enter__(self) -> "PyPIClient": - return self - - def __exit__(self, *exc: object) -> None: - self.close() - - # --- Endpoints --- - - def get_recent_downloads(self, package: str) -> dict: - """`GET /packages/{package}/recent` (last day/week/month totals).""" - return self._request("GET", f"/packages/{package}/recent").json() - - def get_overall_downloads(self, package: str, *, mirrors: bool = False) -> dict: - """`GET /packages/{package}/overall` (daily download time series).""" - params = {"mirrors": str(mirrors).lower()} - return self._request( - "GET", f"/packages/{package}/overall", params=params - ).json() - - # --- Internals --- - - def _request( - self, method: str, url: str, params: dict[str, object] | None = None - ) -> httpx.Response: - """Issue one request, retrying transient failures, then pause politely.""" - response = self._client.request(method, url, params=params) - for attempt in range(self._max_retries): - if response.status_code not in RETRYABLE_STATUSES: - break - self._sleep(retry_delay(response, attempt, self._backoff_base)) - response = self._client.request(method, url, params=params) - response.raise_for_status() - self._sleep(self._min_interval) - return response diff --git a/src/repolytics/ingestion/pypi_source.py b/src/repolytics/ingestion/pypi_source.py new file mode 100644 index 0000000..6edd5e6 --- /dev/null +++ b/src/repolytics/ingestion/pypi_source.py @@ -0,0 +1,36 @@ +"""dlt source for the PyPI Stats API. + +Fetches the per-package overall download time series and yields one row per +day/category, stamped with `_package` + `_loaded_at`. +""" + +import time +from collections.abc import Iterator + +import dlt +from dlt.sources.helpers.rest_client import RESTClient + +from repolytics.ingestion._meta import stamp + +BASE_URL = "https://pypistats.org/api" + + +@dlt.source(name="pypi") +def pypi_source(packages: list[str], *, min_interval: float = 1.0) -> object: + """dlt source yielding PyPI overall downloads for each package.""" + client = RESTClient(base_url=BASE_URL) + + @dlt.resource( + name="downloads", + write_disposition="merge", + primary_key=["_package", "category", "date"], + ) + def downloads() -> Iterator[dict]: + for package in packages: + response = client.get(f"/packages/{package}/overall") + response.raise_for_status() + apply = stamp(_package=package) + yield from (apply(row) for row in response.json()["data"]) + time.sleep(min_interval) # courtesy delay between packages + + return downloads diff --git a/src/repolytics/ingestion/watermarks.py b/src/repolytics/ingestion/watermarks.py deleted file mode 100644 index 50c6294..0000000 --- a/src/repolytics/ingestion/watermarks.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Watermark store for incremental extraction. - -Tracks the last-seen timestamp per `{source}/{endpoint}/{repo}` as a flat JSON -map on disk, so each ingestion run only fetches records newer than last time. -""" - -import json -from pathlib import Path - - -def watermark_key(source: str, endpoint: str, repo: str) -> str: - """Build the watermark map key for a source/endpoint/repo target.""" - return f"{source}/{endpoint}/{repo}" - - -def load_watermarks(path: str | Path) -> dict[str, str]: - """Load the watermark map, returning an empty dict when the file is absent.""" - file = Path(path) - if not file.exists(): - return {} - return json.loads(file.read_text(encoding="utf-8")) - - -def save_watermarks(watermarks: dict[str, str], path: str | Path) -> None: - """Atomically write the watermark map to `path`, creating parents as needed.""" - file = Path(path) - file.parent.mkdir(parents=True, exist_ok=True) - tmp = file.with_name(f"{file.name}.tmp") - tmp.write_text(json.dumps(watermarks, indent=2), encoding="utf-8") - tmp.replace(file) diff --git a/src/repolytics/ingestion/writer.py b/src/repolytics/ingestion/writer.py deleted file mode 100644 index 74bb2f4..0000000 --- a/src/repolytics/ingestion/writer.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Parquet writer for raw API responses. - -Lands each record as a single JSON `data` column plus a `_loaded_at` timestamp. -Optional extraction metadata (e.g. the source repo) is stamped as extra -`_`-prefixed string columns, kept separate from the pristine `data` blob. -""" - -import json -from datetime import UTC, datetime -from pathlib import Path - -import polars as pl - -# Raw-landing schema; metadata columns are appended per call. -_SCHEMA = {"data": pl.Utf8, "_loaded_at": pl.Datetime(time_unit="us", time_zone="UTC")} - - -def write_parquet( - records: list[dict], - path: str | Path, - *, - metadata: dict[str, str] | None = None, -) -> Path: - """Write `records` to `path` as Parquet (JSON `data` column + `_loaded_at`). - - Each record is serialized to a JSON string and stamped with a single batch - timestamp. `metadata` adds one constant string column per key to every row - (extraction provenance such as the source repo), distinct from the payload. - An empty `records` list still writes a zero-row file with the full schema, - so the partition stays present and schema-stable. - """ - metadata = metadata or {} - out = Path(path) - out.parent.mkdir(parents=True, exist_ok=True) - loaded_at = datetime.now(UTC) - schema = {**_SCHEMA, **dict.fromkeys(metadata, pl.Utf8)} - frame = pl.DataFrame( - { - "data": [json.dumps(record, ensure_ascii=False) for record in records], - "_loaded_at": [loaded_at] * len(records), - **{key: [value] * len(records) for key, value in metadata.items()}, - }, - schema=schema, - ) - frame.write_parquet(out) - return out - - -def partition_path( - root: str | Path, - source: str, - table: str, - date: datetime | str, - *, - entity: str | None = None, -) -> Path: - """Build the date-partitioned landing path for a source/table on a date. - - When `entity` is given (e.g. a repo or package), it is inserted before - the date and its `/` slugified to `__`, so each entity lands in its own - file without collisions. - - Returns `{root}/{source}/{table}/{YYYY-MM-DD}.parquet`. - """ - day = date.strftime("%Y-%m-%d") if isinstance(date, datetime) else date - base = Path(root) / source / table - if entity is not None: - base = base / entity.replace("/", "__") - return base / f"{day}.parquet" diff --git a/src/repolytics/loading/__init__.py b/src/repolytics/loading/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/repolytics/loading/raw_loader.py b/src/repolytics/loading/raw_loader.py deleted file mode 100644 index 622ee35..0000000 --- a/src/repolytics/loading/raw_loader.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Load landed Parquet into the DuckDB `raw` schema.""" - -from pathlib import Path - -import duckdb - -from repolytics.config import Settings, get_settings - -RAW_SCHEMA = "raw" - - -def load_raw(conn: duckdb.DuckDBPyConnection, raw_root: str | Path) -> list[str]: - """Load every `{source}/{table}` partition under `raw_root` into `raw.{table}`. - - Tables are (re)created with `CREATE OR REPLACE`, so the load is idempotent. - `union_by_name` tolerates partitions written with or without optional - metadata columns. Returns the sorted list of loaded table names. - """ - conn.execute(f"CREATE SCHEMA IF NOT EXISTS {RAW_SCHEMA}") - loaded: list[str] = [] - for table_dir in _table_dirs(Path(raw_root)): - table = table_dir.name - pattern = (table_dir / "**" / "*.parquet").as_posix() - conn.execute( - f'CREATE OR REPLACE TABLE {RAW_SCHEMA}."{table}" AS ' - "SELECT * FROM read_parquet(?, union_by_name = true)", - [pattern], - ) - loaded.append(table) - return loaded - - -def _table_dirs(root: Path) -> list[Path]: - """Return sorted `{source}/{table}` dirs under `root` that hold Parquet.""" - if not root.exists(): - return [] - return sorted( - table_dir - for source_dir in root.iterdir() - if source_dir.is_dir() - for table_dir in source_dir.iterdir() - if table_dir.is_dir() and next(table_dir.glob("**/*.parquet"), None) - ) - - -def load_all(settings: Settings | None = None) -> list[str]: - """Open the configured DuckDB file and load all raw partitions into it.""" - settings = settings or get_settings() - settings.duckdb_path.parent.mkdir(parents=True, exist_ok=True) - conn = duckdb.connect(str(settings.duckdb_path)) - try: - return load_raw(conn, settings.raw_data_path) - finally: - conn.close() diff --git a/tests/conftest.py b/tests/conftest.py index d3e9433..5642bbf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,13 +1,9 @@ """Shared pytest fixtures.""" -from collections.abc import Iterator from pathlib import Path -import duckdb import pytest -from repolytics.config import Settings - @pytest.fixture def tmp_data_dir(tmp_path: Path) -> Path: @@ -20,27 +16,5 @@ def tmp_data_dir(tmp_path: Path) -> Path: @pytest.fixture def tmp_duckdb_path(tmp_data_dir: Path) -> Path: - """Path to a (not-yet-created) DuckDB file inside the temp data tree.""" + """Path to a DuckDB file inside the temp data tree.""" return tmp_data_dir / "warehouse" / "test.duckdb" - - -@pytest.fixture -def duckdb_conn() -> Iterator[duckdb.DuckDBPyConnection]: - """An in-memory DuckDB connection, closed on teardown.""" - conn = duckdb.connect(":memory:") - try: - yield conn - finally: - conn.close() - - -@pytest.fixture -def settings(tmp_data_dir: Path) -> Settings: - """A `Settings` instance pointed at the temp data tree.""" - return Settings( - github_token="test-token", - github_target_repos="owner/repo-a, owner/repo-b", - duckdb_path=tmp_data_dir / "warehouse" / "repolytics.duckdb", - raw_data_path=tmp_data_dir / "raw", - watermarks_path=tmp_data_dir / "raw" / ".watermarks.json", - ) diff --git a/tests/fixtures/github_responses/commits.json b/tests/fixtures/github_responses/commits.json index 3de7ca4..ab36486 100644 --- a/tests/fixtures/github_responses/commits.json +++ b/tests/fixtures/github_responses/commits.json @@ -6,7 +6,7 @@ "message": "fix: handle empty response" }, "author": {"login": "alice", "id": 1}, - "stats": {"additions": 10, "deletions": 2, "total": 12} + "parents": [{"sha": "aaa000", "url": "https://api.github.com/repos/encode/httpx/commits/aaa000"}] }, { "sha": "def456", @@ -15,6 +15,6 @@ "message": "feat: add retry support" }, "author": {"login": "bob", "id": 2}, - "stats": {"additions": 30, "deletions": 5, "total": 35} + "parents": [{"sha": "abc123", "url": "https://api.github.com/repos/encode/httpx/commits/abc123"}] } ] diff --git a/tests/fixtures/github_responses/pulls.json b/tests/fixtures/github_responses/pulls.json index 05f79e1..5d8d386 100644 --- a/tests/fixtures/github_responses/pulls.json +++ b/tests/fixtures/github_responses/pulls.json @@ -7,10 +7,6 @@ "created_at": "2024-01-02T00:00:00Z", "updated_at": "2024-01-06T00:00:00Z", "merged_at": "2024-01-06T00:00:00Z", - "additions": 120, - "deletions": 30, - "review_comments": 5, - "comments": 3, "labels": [{"id": 2, "name": "enhancement", "color": "a2eeef"}] }, { @@ -21,10 +17,6 @@ "created_at": "2024-01-04T00:00:00Z", "updated_at": "2024-01-04T00:00:00Z", "merged_at": null, - "additions": 10, - "deletions": 1, - "review_comments": 0, - "comments": 0, "labels": [] } ] diff --git a/tests/fixtures/pypi_responses/recent.json b/tests/fixtures/pypi_responses/recent.json deleted file mode 100644 index 86c9d96..0000000 --- a/tests/fixtures/pypi_responses/recent.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "data": {"last_day": 12345, "last_week": 80000, "last_month": 350000}, - "package": "polars", - "type": "recent_downloads" -} diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py new file mode 100644 index 0000000..91ff378 --- /dev/null +++ b/tests/integration/test_pipeline.py @@ -0,0 +1,130 @@ +"""Integration test: dlt normalizes the fixtures into the DuckDB `raw` schema. + +Feeds the recorded API fixtures through dlt (using the real `stamp` metadata helper) +into a temporary DuckDB and asserts the normalized table/column contract that the dbt +staging models depend on. Also checks idempotency of the merge write disposition. +""" + +import json +from collections.abc import Iterator +from pathlib import Path + +import dlt +import duckdb +import pytest + +from repolytics.ingestion._meta import stamp + +FIXTURES = Path(__file__).parent.parent / "fixtures" +REPO = "encode/httpx" +PACKAGE = "httpx" # matches the repo fixture so the downloads->repo FK resolves + + +def _read(rel: str) -> object: + return json.loads((FIXTURES / rel).read_text(encoding="utf-8")) + + +def _sources() -> list: + @dlt.resource(name="repositories", write_disposition="merge", primary_key="id") + def repositories() -> Iterator[dict]: + yield stamp(_repo=REPO)(_read("github_responses/repository.json")) + + @dlt.resource(name="commits", write_disposition="merge", primary_key="sha") + def commits() -> Iterator[dict]: + rows = _read("github_responses/commits.json") + yield from (stamp(_repo=REPO)(c) for c in rows) + + @dlt.resource( + name="issues", write_disposition="merge", primary_key=["_repo", "number"] + ) + def issues() -> Iterator[dict]: + yield from (stamp(_repo=REPO)(c) for c in _read("github_responses/issues.json")) + + @dlt.resource( + name="pull_requests", write_disposition="merge", primary_key=["_repo", "number"] + ) + def pull_requests() -> Iterator[dict]: + yield from (stamp(_repo=REPO)(c) for c in _read("github_responses/pulls.json")) + + @dlt.resource(name="releases", write_disposition="merge", primary_key="id") + def releases() -> Iterator[dict]: + yield from ( + stamp(_repo=REPO)(c) for c in _read("github_responses/releases.json") + ) + + @dlt.resource( + name="downloads", + write_disposition="merge", + primary_key=["_package", "category", "date"], + ) + def downloads() -> Iterator[dict]: + data = _read("pypi_responses/overall.json")["data"] + yield from (stamp(_package=PACKAGE)(r) for r in data) + + return [repositories, commits, issues, pull_requests, releases, downloads] + + +@pytest.fixture +def loaded_db(tmp_duckdb_path: Path) -> Iterator[duckdb.DuckDBPyConnection]: + pipeline = dlt.pipeline( + pipeline_name="repolytics_test", + destination=dlt.destinations.duckdb(str(tmp_duckdb_path)), + dataset_name="raw", + ) + pipeline.run(_sources()) + conn = duckdb.connect(str(tmp_duckdb_path)) + try: + yield conn + finally: + conn.close() + + +def _tables(conn: duckdb.DuckDBPyConnection) -> set[str]: + rows = conn.execute( + "select table_name from information_schema.tables where table_schema = 'raw'" + ).fetchall() + return {r[0] for r in rows} + + +def test_normalized_tables_and_child_tables_exist( + loaded_db: duckdb.DuckDBPyConnection, +) -> None: + assert { + "repositories", + "repositories__topics", + "commits", + "commits__parents", + "issues", + "issues__labels", + "pull_requests", + "pull_requests__labels", + "releases", + "downloads", + } <= _tables(loaded_db) + + +def test_metadata_columns_present(loaded_db: duckdb.DuckDBPyConnection) -> None: + commit_cols = { + r[0] + for r in loaded_db.execute( + "select column_name from information_schema.columns " + "where table_schema = 'raw' and table_name = 'commits'" + ).fetchall() + } + assert {"_repo", "_loaded_at", "commit__author__date"} <= commit_cols + + repo = loaded_db.execute("select distinct _repo from raw.commits").fetchone()[0] + assert repo == REPO + package = loaded_db.execute( + "select distinct _package from raw.downloads" + ).fetchone()[0] + assert package == PACKAGE + + +def test_row_counts(loaded_db: duckdb.DuckDBPyConnection) -> None: + def count(table: str) -> int: + return loaded_db.execute(f"select count(*) from raw.{table}").fetchone()[0] + + assert count("commits") == 2 + assert count("issues") == 2 # includes the PR-shaped issue (staging filters it) + assert count("downloads") == 2 diff --git a/tests/integration/test_raw_loader.py b/tests/integration/test_raw_loader.py deleted file mode 100644 index e27ed3b..0000000 --- a/tests/integration/test_raw_loader.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Integration tests for repolytics.loading.raw_loader.""" - -import json -from pathlib import Path - -import duckdb -import pytest - -from repolytics.ingestion.writer import partition_path, write_parquet -from repolytics.loading.raw_loader import load_raw - -FIXTURES = Path(__file__).parent.parent / "fixtures" -REPO = "encode/httpx" -PACKAGE = "polars" -DAY = "2024-01-02" - -# Source table -> (fixtures file, metadata). -_GITHUB_TABLES = { - "commits": "commits.json", - "issues": "issues.json", - "pull_requests": "pulls.json", - "releases": "releases.json", -} - - -def _read_json(rel: str) -> object: - return json.loads((FIXTURES / rel).read_text(encoding="utf-8")) - - -@pytest.fixture -def raw_root(tmp_data_dir: Path) -> Path: - """A populated `data/raw` tree built from the JSON fixtures via write_parquet.""" - root = tmp_data_dir / "raw" - - repo = _read_json("github_responses/repository.json") - write_parquet( - [repo], - partition_path(root, "github", "repositories", DAY, entity=REPO), - metadata={"_repo": REPO}, - ) - for table, fname in _GITHUB_TABLES.items(): - write_parquet( - _read_json(f"github_responses/{fname}"), - partition_path(root, "github", table, DAY, entity=REPO), - metadata={"_repo": REPO}, - ) - overall = _read_json("pypi_responses/overall.json") - write_parquet( - overall["data"], - partition_path(root, "pypi", "downloads", DAY, entity=PACKAGE), - metadata={"_package": PACKAGE}, - ) - return root - - -def test_load_raw_creates_expected_tables( - raw_root: Path, duckdb_conn: duckdb.DuckDBPyConnection -) -> None: - loaded = load_raw(duckdb_conn, raw_root) - - assert set(loaded) == { - "repositories", - "commits", - "issues", - "pull_requests", - "releases", - "downloads", - } - schemas = duckdb_conn.execute( - "SELECT schema_name FROM information_schema.schemata" - ).fetchall() - assert "raw" in {row[0] for row in schemas} - - -def test_load_raw_row_counts( - raw_root: Path, duckdb_conn: duckdb.DuckDBPyConnection -) -> None: - load_raw(duckdb_conn, raw_root) - - def count(table: str) -> int: - return duckdb_conn.execute(f"SELECT count(*) FROM raw.{table}").fetchone()[0] - - assert count("repositories") == 1 - assert count("commits") == 2 - assert count("issues") == 2 # includes the PR-shaped issue (staging filters it) - assert count("downloads") == 2 - - -def test_load_raw_preserves_data_and_metadata_columns( - raw_root: Path, duckdb_conn: duckdb.DuckDBPyConnection -) -> None: - load_raw(duckdb_conn, raw_root) - - columns = duckdb_conn.execute( - "SELECT column_name FROM information_schema.columns " - "WHERE table_schema = 'raw' AND table_name = 'commits'" - ).fetchall() - assert {"data", "_loaded_at", "_repo"} <= {row[0] for row in columns} - - repo = duckdb_conn.execute("SELECT DISTINCT _repo FROM raw.commits").fetchone()[0] - assert repo == REPO - package = duckdb_conn.execute( - "SELECT DISTINCT _package FROM raw.downloads" - ).fetchone()[0] - assert package == PACKAGE - - -def test_load_raw_is_idempotent( - raw_root: Path, duckdb_conn: duckdb.DuckDBPyConnection -) -> None: - load_raw(duckdb_conn, raw_root) - load_raw(duckdb_conn, raw_root) # second run must not duplicate rows - - count = duckdb_conn.execute("SELECT count(*) FROM raw.commits").fetchone()[0] - assert count == 2 - - -def test_load_raw_empty_root_returns_nothing( - tmp_data_dir: Path, duckdb_conn: duckdb.DuckDBPyConnection -) -> None: - assert load_raw(duckdb_conn, tmp_data_dir / "raw") == [] diff --git a/tests/unit/_clients.py b/tests/unit/_clients.py deleted file mode 100644 index 437ea77..0000000 --- a/tests/unit/_clients.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Builders for mock-transport-backed API clients.""" - -from collections.abc import Callable - -import httpx - - -def mock_client( - client_cls: type, - handler: Callable[[httpx.Request], httpx.Response], - *args: object, - **kwargs: object, -) -> tuple[object, list[float]]: - """Build an ingestion client over a `MockTransport` with a spy `sleep`. - - Returns the client and the list recording every sleep duration, so tests can - assert on retry/courtesy waits without blocking. - """ - sleeps: list[float] = [] - client = client_cls( - *args, - transport=httpx.MockTransport(handler), - sleep=sleeps.append, - backoff_base=0.0, - **kwargs, - ) - return client, sleeps diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 0112f21..2701894 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -1,4 +1,4 @@ -"""Tests for repolytics.config.Settings.""" +"""Unit tests for repolytics.config.Settings.""" from pathlib import Path @@ -7,16 +7,21 @@ from repolytics.config import Settings +_PROJECTS_CSV = """\ +repo,package +fastapi/fastapi,fastapi +pola-rs/polars,polars +torvalds/linux, +""" + def test_loads_from_environment(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("GITHUB_TOKEN", "ghp_secret") - monkeypatch.setenv("GITHUB_TARGET_REPOS", "fastapi/fastapi,pola-rs/polars") monkeypatch.setenv("DUCKDB_PATH", "custom/wh.duckdb") settings = Settings(_env_file=None) assert settings.github_token.get_secret_value() == "ghp_secret" - assert settings.github_target_repos == "fastapi/fastapi,pola-rs/polars" assert settings.duckdb_path == Path("custom/wh.duckdb") @@ -35,31 +40,41 @@ def test_token_is_not_exposed_in_repr(monkeypatch: pytest.MonkeyPatch) -> None: assert "ghp_secret" not in repr(settings) -def test_target_repos_parses_and_strips(monkeypatch: pytest.MonkeyPatch) -> None: +def test_projects_derive_repos_and_packages( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: monkeypatch.setenv("GITHUB_TOKEN", "ghp_secret") - monkeypatch.setenv("GITHUB_TARGET_REPOS", " a/b , c/d ,, e/f ") + projects_file = tmp_path / "projects.csv" + projects_file.write_text(_PROJECTS_CSV, encoding="utf-8") - settings = Settings(_env_file=None) + settings = Settings(_env_file=None, projects_file=projects_file) - assert settings.target_repos == ["a/b", "c/d", "e/f"] + assert settings.target_repos == [ + "fastapi/fastapi", + "pola-rs/polars", + "torvalds/linux", + ] + # The project with a blank `package` is skipped for PyPI. + assert settings.packages == ["fastapi", "polars"] -def test_target_repos_empty_when_unset(monkeypatch: pytest.MonkeyPatch) -> None: +def test_projects_empty_when_file_absent( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: monkeypatch.setenv("GITHUB_TOKEN", "ghp_secret") - monkeypatch.delenv("GITHUB_TARGET_REPOS", raising=False) - settings = Settings(_env_file=None) + settings = Settings(_env_file=None, projects_file=tmp_path / "missing.csv") + assert settings.projects == [] assert settings.target_repos == [] + assert settings.packages == [] def test_defaults_apply(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("GITHUB_TOKEN", "ghp_secret") - for var in ("DUCKDB_PATH", "RAW_DATA_PATH", "WATERMARKS_PATH"): - monkeypatch.delenv(var, raising=False) + monkeypatch.delenv("DUCKDB_PATH", raising=False) settings = Settings(_env_file=None) assert settings.duckdb_path == Path("data/warehouse/repolytics.duckdb") - assert settings.raw_data_path == Path("data/raw") - assert settings.watermarks_path == Path("data/raw/.watermarks.json") + assert settings.projects_file == Path("dbt/seeds/projects.csv") diff --git a/tests/unit/test_github_client.py b/tests/unit/test_github_client.py deleted file mode 100644 index eca5da0..0000000 --- a/tests/unit/test_github_client.py +++ /dev/null @@ -1,250 +0,0 @@ -"""Tests for repolytics.ingestion.github_client.GitHubClient.""" - -import time -from collections.abc import Callable -from datetime import UTC, datetime - -import httpx -import pytest - -from repolytics.ingestion._http import RETRYABLE_STATUSES -from repolytics.ingestion.github_client import GitHubClient, _to_iso -from tests.unit._clients import mock_client - -# Absolute next-page URL used to drive pagination in tests. -NEXT_URL = "https://api.github.com/x?page=2" -NEXT_LINK = f'<{NEXT_URL}>; rel="next"' - - -def make_client( - handler: Callable[[httpx.Request], httpx.Response], **kwargs: object -) -> tuple[GitHubClient, list[float]]: - """Build a `GitHubClient` over a `MockTransport` with a spy `sleep`.""" - return mock_client(GitHubClient, handler, "test-token", **kwargs) - - -@pytest.mark.parametrize("dt", [datetime(2024, 1, 1), datetime(2024, 1, 1, tzinfo=UTC)]) -def test_to_iso_appends_z(dt: datetime) -> None: - assert _to_iso(dt) == "2024-01-01T00:00:00Z" - - -def test_paginates_following_next_link() -> None: - calls: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - calls.append(request) - if len(calls) == 1: - return httpx.Response( - 200, json=[{"id": 1}, {"id": 2}], headers={"Link": NEXT_LINK} - ) - return httpx.Response(200, json=[{"id": 3}]) - - client, _ = make_client(handler) - result = client.get_commits("o", "r") - - assert [c["id"] for c in result] == [1, 2, 3] - assert len(calls) == 2 - assert str(calls[1].url) == NEXT_URL - - -def test_sleeps_when_rate_limit_low() -> None: - reset = int(time.time()) + 30 - - def handler(request: httpx.Request) -> httpx.Response: - return httpx.Response( - 200, - json={"id": 1}, - headers={ - "X-RateLimit-Remaining": "0", - "X-RateLimit-Reset": str(reset), - }, - ) - - client, sleeps = make_client(handler) - client.get_repository("o", "r") - - assert len(sleeps) == 1 - assert sleeps[0] > 0 - - -@pytest.mark.parametrize("status", sorted(RETRYABLE_STATUSES)) -def test_retryable_status_retries_then_succeeds(status: int) -> None: - statuses = [status, 200] - calls: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - current = statuses[len(calls)] - calls.append(request) - if current == 200: - return httpx.Response(200, json={"id": 1}) - return httpx.Response(current) - - client, sleeps = make_client(handler) - result = client.get_repository("o", "r") - - assert result == {"id": 1} - assert len(calls) == 2 # one failure, one retry - assert len(sleeps) == 1 - - -def test_raises_after_max_retries() -> None: - def handler(request: httpx.Request) -> httpx.Response: - return httpx.Response(502) - - client, sleeps = make_client(handler, max_retries=3) - with pytest.raises(httpx.HTTPStatusError): - client.get_repository("o", "r") - - assert len(sleeps) == 3 - - -@pytest.mark.parametrize("status", [400, 401, 403, 404]) -def test_non_retryable_status_raises_without_retry(status: int) -> None: - calls: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - calls.append(request) - return httpx.Response(status) - - client, sleeps = make_client(handler) - with pytest.raises(httpx.HTTPStatusError): - client.get_repository("o", "r") - - assert len(calls) == 1 - assert sleeps == [] - - -def test_429_honors_retry_after() -> None: - statuses = [429, 200] - calls: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - status = statuses[len(calls)] - calls.append(request) - if status == 429: - return httpx.Response(429, headers={"Retry-After": "2"}) - return httpx.Response(200, json={"id": 1}) - - client, sleeps = make_client(handler) - result = client.get_repository("o", "r") - - assert result == {"id": 1} - assert sleeps == [2.0] - - -def test_get_repository_returns_raw_json() -> None: - payload = { - "id": 123, - "name": "httpx", - "owner": {"login": "encode"}, - "stargazers_count": 1000, - "license": {"spdx_id": "BSD-3-Clause"}, - } - - def handler(request: httpx.Request) -> httpx.Response: - return httpx.Response(200, json=payload) - - client, _ = make_client(handler) - assert client.get_repository("encode", "httpx") == payload - - -def test_get_commits_passes_since() -> None: - captured: dict[str, str | None] = {} - - def handler(request: httpx.Request) -> httpx.Response: - captured["since"] = request.url.params.get("since") - return httpx.Response(200, json=[]) - - client, _ = make_client(handler) - client.get_commits("o", "r", since=datetime(2024, 1, 1, tzinfo=UTC)) - - assert captured["since"] == "2024-01-01T00:00:00Z" - - -def test_get_issues_sets_state_and_since() -> None: - captured: dict[str, str | None] = {} - - def handler(request: httpx.Request) -> httpx.Response: - captured["state"] = request.url.params.get("state") - captured["since"] = request.url.params.get("since") - return httpx.Response(200, json=[]) - - client, _ = make_client(handler) - client.get_issues("o", "r", since=datetime(2024, 1, 1, tzinfo=UTC)) - - assert captured == {"state": "all", "since": "2024-01-01T00:00:00Z"} - - -def test_get_releases_returns_list() -> None: - def handler(request: httpx.Request) -> httpx.Response: - return httpx.Response(200, json=[{"tag_name": "v1"}]) - - client, _ = make_client(handler) - assert client.get_releases("o", "r") == [{"tag_name": "v1"}] - - -def test_get_pull_requests_without_since_fetches_all() -> None: - captured: dict[str, str | None] = {} - - def handler(request: httpx.Request) -> httpx.Response: - captured["state"] = request.url.params.get("state") - captured["sort"] = request.url.params.get("sort") - return httpx.Response(200, json=[{"number": 1}]) - - client, _ = make_client(handler) - result = client.get_pull_requests("o", "r") - - assert result == [{"number": 1}] - assert captured["state"] == "all" - assert captured["sort"] is None - - -def test_context_manager_closes_client() -> None: - def handler(request: httpx.Request) -> httpx.Response: - return httpx.Response(200, json={"id": 1}) - - client, _ = make_client(handler) - with client as c: - assert c.get_repository("o", "r") == {"id": 1} - assert client._client.is_closed - - -def test_from_settings_uses_env_token(monkeypatch: pytest.MonkeyPatch) -> None: - from repolytics.config import get_settings - - monkeypatch.setenv("GITHUB_TOKEN", "ghp_from_env") - get_settings.cache_clear() - - def handler(request: httpx.Request) -> httpx.Response: - assert request.headers["Authorization"] == "Bearer ghp_from_env" - return httpx.Response(200, json={"id": 1}) - - client = GitHubClient.from_settings( - transport=httpx.MockTransport(handler), sleep=lambda _: None - ) - try: - assert client.get_repository("o", "r") == {"id": 1} - finally: - client.close() - get_settings.cache_clear() - - -def test_get_pull_requests_early_exits_past_since() -> None: - page = [ - {"number": 3, "updated_at": "2024-03-01T00:00:00Z"}, - {"number": 2, "updated_at": "2024-02-01T00:00:00Z"}, - {"number": 1, "updated_at": "2023-12-01T00:00:00Z"}, - ] - calls: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - calls.append(request) - return httpx.Response(200, json=page, headers={"Link": NEXT_LINK}) - - client, _ = make_client(handler) - result = client.get_pull_requests("o", "r", since=datetime(2024, 1, 1, tzinfo=UTC)) - - assert [pr["number"] for pr in result] == [3, 2] - assert len(calls) == 1 # stopped early, did not follow the next link - assert calls[0].url.params.get("sort") == "updated" - assert calls[0].url.params.get("direction") == "desc" diff --git a/tests/unit/test_meta.py b/tests/unit/test_meta.py new file mode 100644 index 0000000..b99e4d2 --- /dev/null +++ b/tests/unit/test_meta.py @@ -0,0 +1,22 @@ +"""Unit tests for the repolytics.ingestion._meta.stamp""" + +from datetime import datetime + +from repolytics.ingestion._meta import stamp + + +def test_stamp_adds_loaded_at_and_metadata() -> None: + record = {"id": 1, "name": "x"} + stamped = stamp(_repo="encode/httpx")(record) + + assert stamped["id"] == 1 + assert stamped["name"] == "x" + assert stamped["_repo"] == "encode/httpx" + assert isinstance(stamped["_loaded_at"], datetime) + + +def test_stamp_does_not_mutate_input() -> None: + record = {"id": 1} + stamp(_package="polars")(record) + + assert record == {"id": 1} diff --git a/tests/unit/test_pypi_client.py b/tests/unit/test_pypi_client.py deleted file mode 100644 index 4dceccf..0000000 --- a/tests/unit/test_pypi_client.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Tests for repolytics.ingestion.pypi_client.PyPIClient.""" - -from collections.abc import Callable - -import httpx -import pytest - -from repolytics.ingestion._http import RETRYABLE_STATUSES -from repolytics.ingestion.pypi_client import PyPIClient -from tests.unit._clients import mock_client - - -def make_client( - handler: Callable[[httpx.Request], httpx.Response], **kwargs: object -) -> tuple[PyPIClient, list[float]]: - """Build a `PyPIClient` over a `MockTransport` with a spy `sleep`.""" - return mock_client(PyPIClient, handler, **kwargs) - - -def test_get_recent_downloads_returns_raw_json() -> None: - payload = { - "data": {"last_day": 1, "last_week": 2, "last_month": 3}, - "package": "polars", - "type": "recent_downloads", - } - - def handler(request: httpx.Request) -> httpx.Response: - assert request.url.path == "/api/packages/polars/recent" - return httpx.Response(200, json=payload) - - client, _ = make_client(handler, min_interval=0.0) - assert client.get_recent_downloads("polars") == payload - - -def test_get_overall_sets_mirrors_param() -> None: - captured: dict[str, str | None] = {} - - def handler(request: httpx.Request) -> httpx.Response: - captured["path"] = request.url.path - captured["mirrors"] = request.url.params.get("mirrors") - return httpx.Response(200, json={"data": []}) - - client, _ = make_client(handler, min_interval=0.0) - client.get_overall_downloads("polars", mirrors=True) - - assert captured["path"] == "/api/packages/polars/overall" - assert captured["mirrors"] == "true" - - -def test_courtesy_delay_applied() -> None: - def handler(request: httpx.Request) -> httpx.Response: - return httpx.Response(200, json={"data": {}}) - - client, sleeps = make_client(handler, min_interval=1.0) - client.get_recent_downloads("polars") - - assert sleeps == [1.0] - - -@pytest.mark.parametrize("status", sorted(RETRYABLE_STATUSES)) -def test_retryable_status_retries_then_succeeds(status: int) -> None: - statuses = [status, 200] - calls: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - current = statuses[len(calls)] - calls.append(request) - if current == 200: - return httpx.Response(200, json={"data": {}}) - return httpx.Response(current) - - client, sleeps = make_client(handler, min_interval=0.0) - result = client.get_recent_downloads("polars") - - assert result == {"data": {}} - assert len(calls) == 2 - assert len(sleeps) == 2 # one retry backoff + one courtesy delay - - -def test_429_honors_retry_after() -> None: - statuses = [429, 200] - calls: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - status = statuses[len(calls)] - calls.append(request) - if status == 429: - return httpx.Response(429, headers={"Retry-After": "2"}) - return httpx.Response(200, json={"data": {}}) - - client, sleeps = make_client(handler, min_interval=0.0) - result = client.get_recent_downloads("polars") - - assert result == {"data": {}} - assert sleeps == [2.0, 0.0] # Retry-After honored, then courtesy delay - - -def test_context_manager_closes_client() -> None: - def handler(request: httpx.Request) -> httpx.Response: - return httpx.Response(200, json={"data": {}}) - - client, _ = make_client(handler, min_interval=0.0) - with client as c: - assert c.get_recent_downloads("polars") == {"data": {}} - assert client._client.is_closed diff --git a/tests/unit/test_watermarks.py b/tests/unit/test_watermarks.py deleted file mode 100644 index ea56cbf..0000000 --- a/tests/unit/test_watermarks.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Tests for repolytics.ingestion.watermarks.""" - -from pathlib import Path - -from repolytics.ingestion.watermarks import ( - load_watermarks, - save_watermarks, - watermark_key, -) - - -def test_watermark_key_format() -> None: - key = watermark_key("github", "commits", "encode/httpx") - assert key == "github/commits/encode/httpx" - - -def test_load_missing_returns_empty(tmp_path: Path) -> None: - assert load_watermarks(tmp_path / "nope.json") == {} - - -def test_save_then_load_roundtrip(tmp_path: Path) -> None: - path = tmp_path / "wm" / ".watermarks.json" - marks = {"github/commits/encode/httpx": "2024-01-01T00:00:00Z"} - save_watermarks(marks, path) - - assert path.exists() - assert load_watermarks(path) == marks - - -def test_save_creates_parent_dirs(tmp_path: Path) -> None: - path = tmp_path / "deep" / "nested" / ".watermarks.json" - save_watermarks({"a": "b"}, path) - assert path.exists() diff --git a/tests/unit/test_writer.py b/tests/unit/test_writer.py deleted file mode 100644 index 4207e8d..0000000 --- a/tests/unit/test_writer.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Tests for repolytics.ingestion.writer.""" - -import json -from datetime import datetime -from pathlib import Path - -import duckdb -import polars as pl - -from repolytics.ingestion.writer import partition_path, write_parquet - - -def test_write_parquet_roundtrips_via_duckdb( - tmp_path: Path, duckdb_conn: duckdb.DuckDBPyConnection -) -> None: - records = [ - {"id": 1, "owner": {"login": "a"}, "topics": ["x", "y"]}, - {"id": 2, "owner": {"login": "b"}, "license": None}, - ] - out = write_parquet(records, tmp_path / "sub" / "repos.parquet") - - assert out.exists() - rows = duckdb_conn.execute( - "SELECT data, _loaded_at IS NOT NULL AS has_loaded_at " - "FROM read_parquet(?) ORDER BY data", - [str(out)], - ).fetchall() - - assert len(rows) == 2 - loaded = [json.loads(data) for data, _ in rows] - assert {record["id"] for record in loaded} == {1, 2} - assert loaded[0]["owner"]["login"] == "a" # nested structure preserved - assert all(has_loaded_at for _, has_loaded_at in rows) - - -def test_write_parquet_columns(tmp_path: Path) -> None: - out = write_parquet([{"id": 1}], tmp_path / "x.parquet") - frame = pl.read_parquet(out) - assert frame.columns == ["data", "_loaded_at"] - - -def test_write_parquet_empty_writes_schema_only(tmp_path: Path) -> None: - out = write_parquet([], tmp_path / "empty.parquet") - frame = pl.read_parquet(out) - assert frame.columns == ["data", "_loaded_at"] - assert frame.height == 0 - - -def test_write_parquet_metadata_columns(tmp_path: Path) -> None: - records = [{"sha": "a"}, {"sha": "b"}] - out = write_parquet( - records, tmp_path / "commits.parquet", metadata={"_repo": "o/r"} - ) - frame = pl.read_parquet(out) - assert frame.columns == ["data", "_loaded_at", "_repo"] - assert frame["_repo"].to_list() == ["o/r", "o/r"] - - -def test_write_parquet_empty_keeps_metadata_columns(tmp_path: Path) -> None: - out = write_parquet([], tmp_path / "empty.parquet", metadata={"_repo": "o/r"}) - frame = pl.read_parquet(out) - assert frame.columns == ["data", "_loaded_at", "_repo"] - assert frame.height == 0 - - -def test_write_parquet_creates_parent_dirs(tmp_path: Path) -> None: - out = write_parquet([{"id": 1}], tmp_path / "a" / "b" / "c.parquet") - assert out.exists() - - -def test_partition_path_formats_date() -> None: - on_date = datetime(2024, 1, 2) - from_datetime = partition_path("data/raw", "github", "commits", on_date) - assert from_datetime == Path("data/raw/github/commits/2024-01-02.parquet") - - from_string = partition_path("data/raw", "github", "commits", "2024-01-02") - assert from_string == Path("data/raw/github/commits/2024-01-02.parquet") - - -def test_partition_path_with_entity_slugifies_slash() -> None: - path = partition_path( - "data/raw", "github", "commits", "2024-01-02", entity="encode/httpx" - ) - assert path == Path("data/raw/github/commits/encode__httpx/2024-01-02.parquet") diff --git a/uv.lock b/uv.lock index 07e4a83..5a70911 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,12 @@ version = 1 revision = 3 requires-python = ">=3.13" +resolution-markers = [ + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", + "(python_full_version >= '3.14' and platform_python_implementation == 'PyPy') or (python_full_version >= '3.14' and sys_platform == 'emscripten')", + "python_full_version < '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", + "(python_full_version < '3.14' and platform_python_implementation == 'PyPy') or (python_full_version < '3.14' and sys_platform == 'emscripten')", +] [[package]] name = "agate" @@ -29,18 +35,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] -[[package]] -name = "anyio" -version = "4.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "idna" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622, upload-time = "2026-03-24T12:59:09.671Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, -] - [[package]] name = "attrs" version = "26.1.0" @@ -381,6 +375,47 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2b/5f/c52bd1255db763d0cdcb7084d2e90c42119cb229302c56bdf1d0aa78abd2/deepdiff-8.6.2-py3-none-any.whl", hash = "sha256:4d22034a866c3928303a9332c279362f714192d9305bac17c498720d095fd1b4", size = 91979, upload-time = "2026-03-18T17:16:32.171Z" }, ] +[[package]] +name = "dlt" +version = "1.28.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "fsspec" }, + { name = "gitpython" }, + { name = "giturlparse" }, + { name = "humanize" }, + { name = "jsonpath-ng" }, + { name = "orjson", marker = "(python_full_version >= '3.14' and platform_python_implementation == 'PyPy') or (python_full_version >= '3.14' and sys_platform == 'emscripten') or (platform_python_implementation != 'PyPy' and sys_platform != 'emscripten')" }, + { name = "packaging" }, + { name = "pathvalidate" }, + { name = "pendulum" }, + { name = "pluggy" }, + { name = "pytz" }, + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "requirements-parser" }, + { name = "rich-argparse" }, + { name = "semver" }, + { name = "setuptools" }, + { name = "simplejson" }, + { name = "sqlglot" }, + { name = "tenacity" }, + { name = "tomlkit" }, + { name = "typing-extensions" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/e9/45803a93a15e901ecb29c9ecbb63e346166b6c485df123f0fa9ff482b042/dlt-1.28.0.tar.gz", hash = "sha256:8fad65ca4c2a7c74b0b78d87ebed3f974cf9ed635e6f662bf4e8dec4be14f53f", size = 1085533, upload-time = "2026-06-15T16:14:52.893Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/73/522146bd24ca8e610b70b7ca939a2d4cbf6a9c6a124cd12063a7e290fd05/dlt-1.28.0-py3-none-any.whl", hash = "sha256:47673d39557d6a80fb02552e8ff00176a23019e7082a28e3469ac76b8c924430", size = 1363253, upload-time = "2026-06-15T16:14:55.413Z" }, +] + +[package.optional-dependencies] +duckdb = [ + { name = "duckdb" }, +] + [[package]] name = "duckdb" version = "1.5.3" @@ -404,40 +439,54 @@ wheels = [ ] [[package]] -name = "h11" -version = "0.16.0" +name = "fsspec" +version = "2026.6.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +sdist = { url = "https://files.pythonhosted.org/packages/10/a1/ae4e3e5003468d6391d2c77b6fa1cd73bd5d13511d81c642d7b28ac90ed4/fsspec-2026.6.0.tar.gz", hash = "sha256:f5bac145310fe30e16e1471bd6840b2d990d609e872251d7e674241822abf01a", size = 313646, upload-time = "2026-06-16T01:57:28.105Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, + { url = "https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl", hash = "sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1", size = 203949, upload-time = "2026-06-16T01:57:26.358Z" }, ] [[package]] -name = "httpcore" -version = "1.0.9" +name = "gitdb" +version = "4.0.12" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "certifi" }, - { name = "h11" }, + { name = "smmap" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, + { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" }, ] [[package]] -name = "httpx" -version = "0.28.1" +name = "gitpython" +version = "3.1.50" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, - { name = "certifi" }, - { name = "httpcore" }, - { name = "idna" }, + { name = "gitdb" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/33/f6/354ae6491228b5eb40e10d89c4d13c651fe1cf7556e35ebdded50cff57ce/gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc", size = 219798, upload-time = "2026-05-06T04:01:26.571Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/7a/1c6e3562dfd8950adbb11ffbc65d21e7c89d01a6e4f137fa981056de25c5/gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9", size = 212507, upload-time = "2026-05-06T04:01:23.799Z" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } + +[[package]] +name = "giturlparse" +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/8a/b70b84cc78f9059d627f09560c81a11e8f046570d220a24e169489cad6c9/giturlparse-0.15.0.tar.gz", hash = "sha256:9af3f1fd5c4a0cac94ddb283593635005646393ee0debbe330d1bdff8866bf2c", size = 16138, upload-time = "2026-06-16T07:28:56.021Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, + { url = "https://files.pythonhosted.org/packages/c7/96/147a2771ab655b9353781fb2f95c94eaf1d8576dccc991c1a61d0d355067/giturlparse-0.15.0-py2.py3-none-any.whl", hash = "sha256:76d2e6983b037356ab99b30683e533ac3db96409b68e2163a20fc3aff6446f10", size = 16683, upload-time = "2026-06-16T07:28:55.184Z" }, +] + +[[package]] +name = "humanize" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/66/a3921783d54be8a6870ac4ccffcd15c4dc0dd7fcce51c6d63b8c63935276/humanize-4.15.0.tar.gz", hash = "sha256:1dd098483eb1c7ee8e32eb2e99ad1910baefa4b75c3aff3a82f4d78688993b10", size = 83599, upload-time = "2025-12-20T20:16:13.19Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/7b/bca5613a0c3b542420cf92bd5e5fb8ebd5435ce1011a091f66bb7693285e/humanize-4.15.0-py3-none-any.whl", hash = "sha256:b1186eb9f5a9749cd9cb8565aee77919dd7c8d076161cf44d70e59e3301e1769", size = 132203, upload-time = "2025-12-20T20:16:11.67Z" }, ] [[package]] @@ -491,6 +540,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "jsonpath-ng" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ply" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/86/08646239a313f895186ff0a4573452038eed8c86f54380b3ebac34d32fb2/jsonpath-ng-1.7.0.tar.gz", hash = "sha256:f6f5f7fd4e5ff79c785f1573b394043b39849fb2bb47bcead935d12b00beab3c", size = 37838, upload-time = "2024-10-11T15:41:42.404Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/5a/73ecb3d82f8615f32ccdadeb9356726d6cae3a4bbc840b437ceb95708063/jsonpath_ng-1.7.0-py3-none-any.whl", hash = "sha256:f3d7f9e848cba1b6da28c55b1c26ff915dc9e0b1ba7e752a53d6da8d5cbd00b6", size = 30105, upload-time = "2024-11-20T17:58:30.418Z" }, +] + [[package]] name = "jsonschema" version = "4.26.0" @@ -527,6 +588,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/d4/c4dcb02ed11f8884e169b3350fc40aa4c08edf8bed77a8f0f267542e6452/leather-0.4.1-py3-none-any.whl", hash = "sha256:ec61cba1ca3ccb96ed90e38b116fc58757d97d352171006b3288c47ce3fbd183", size = 30340, upload-time = "2025-12-15T19:01:40.823Z" }, ] +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -596,6 +669,15 @@ msgpack = [ { name = "msgpack" }, ] +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + [[package]] name = "more-itertools" version = "10.8.0" @@ -664,6 +746,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/27/fb8d7338b4d551900fa3e580acbe7a0cf655d940e164cb5c00ec31961094/orderly_set-5.5.0-py3-none-any.whl", hash = "sha256:46f0b801948e98f427b412fcabb831677194c05c3b699b80de260374baa0b1e7", size = 13068, upload-time = "2025-07-10T20:10:54.377Z" }, ] +[[package]] +name = "orjson" +version = "3.11.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/0c/964746fcafbd16f8ff53219ad9f6b412b34f345c75f384ad434ceaadb538/orjson-3.11.9.tar.gz", hash = "sha256:4fef17e1f8722c11587a6ef18e35902450221da0028e65dbaaa543619e68e48f", size = 5599163, upload-time = "2026-05-06T15:11:08.309Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/33/93fcc25907235c344ae73122f8a4e01d2d393ef062b4af7d2e2487a32c37/orjson-3.11.9-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:4bab1b2d6141fe7b32ae71dac905666ece4f94936efbfb13d55bb7739a3a6021", size = 228458, upload-time = "2026-05-06T15:10:20.079Z" }, + { url = "https://files.pythonhosted.org/packages/8f/27/b1e6dadb3c080313c03fdd8067b85e6a0460c7d8d6a1c3984ef77b904e4d/orjson-3.11.9-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:844417969855fc7a41be124aafe83dc424592a7f77cd4501900c67307122b92c", size = 128368, upload-time = "2026-05-06T15:10:21.549Z" }, + { url = "https://files.pythonhosted.org/packages/21/0f/c9ede0bf052f6b4051e64a7d4fa91b725cccf8321a6a786e86eb03519f00/orjson-3.11.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffe02797b5e9f3a9d8292ddcd289b474ad13e81ad83cd1891a240811f1d2cb81", size = 132070, upload-time = "2026-05-06T15:10:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/fd/26/d398e28048dc18205bbe812f2c88cb9b40313db2470778e25964796458fe/orjson-3.11.9-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e4eed3b200023042814d2fc8a5d2e880f13b52e1ed2485e83da4f3962f7dc1a", size = 127892, upload-time = "2026-05-06T15:10:24.714Z" }, + { url = "https://files.pythonhosted.org/packages/66/60/52b0054c4c700d5aa7fc5b7ca96917400d8f061307778578e67a10e25852/orjson-3.11.9-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aff7da9952a5ad1cef8e68017724d96c7b9a66e99e91d6252e1b133d67a7b10", size = 135217, upload-time = "2026-05-06T15:10:26.084Z" }, + { url = "https://files.pythonhosted.org/packages/d5/97/1e3dc2b2a28b7b2528f403d2fc1d79ec5f39af3bc143ab65d3ec26426385/orjson-3.11.9-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4d4e98d6f3b8afed8bc8cd9718ec0cdf46661826beefb53fe8eafb37f2bf0362", size = 145980, upload-time = "2026-05-06T15:10:28.062Z" }, + { url = "https://files.pythonhosted.org/packages/fc/39/31fbfe7850f2de32dee7e7e5c09f26d403ab01e440ac96001c6b01ad3c99/orjson-3.11.9-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a81d52442a7c99b3662333235b3adf96a1715864658b35bb797212be7bddb97", size = 132738, upload-time = "2026-05-06T15:10:29.727Z" }, + { url = "https://files.pythonhosted.org/packages/a1/08/dca0082dd2a194acb93e5457e73455388e2e2ca464a2672449a9ddbb679d/orjson-3.11.9-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e39364e726a8fff737309aff059ff67d8a8c8d5b677be7bb49a8b3e84b7e218", size = 134033, upload-time = "2026-05-06T15:10:31.152Z" }, + { url = "https://files.pythonhosted.org/packages/11/d4/5bdb0626801230139987385554c5d4c42255218ac906525bf4347f22cd95/orjson-3.11.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4fd66214623f1b17501df9f0543bef0b833979ab5b6ded1e1d123222866aa8c9", size = 141492, upload-time = "2026-05-06T15:10:32.641Z" }, + { url = "https://files.pythonhosted.org/packages/fa/88/a21fb53b3ede6703aede6dce4710ed4111e5b201cfa6bbff5e544f9d47d7/orjson-3.11.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8ecc30f10465fa1e0ce13fd01d9e22c316e5053a719a8d915d4545a09a5ff677", size = 415087, upload-time = "2026-05-06T15:10:34.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/57/1b30daf70f0d8180e9a73cefbfbdd99e4bf19eb020466502b01fba7e0e50/orjson-3.11.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:97db4c94a7db398a5bd636273324f0b3fd58b350bbbac8bb380ceb825a9b40f4", size = 148031, upload-time = "2026-05-06T15:10:36.358Z" }, + { url = "https://files.pythonhosted.org/packages/04/83/45fbb6d962e260807f99441db9613cee868ceda4baceda59b3720a563f97/orjson-3.11.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9f78cf8fec5bd627f4082b8dfeac7871b43d7f3274904492a43dab39f18a19a0", size = 136915, upload-time = "2026-05-06T15:10:38.013Z" }, + { url = "https://files.pythonhosted.org/packages/5f/cc/2d10025f9056d376e4127ec05a5808b218d46f035fdc08178a5411b34250/orjson-3.11.9-cp313-cp313-win32.whl", hash = "sha256:d4087e5c0209a0a8efe4de3303c234b9c44d1174161dcd851e8eea07c7560b32", size = 131613, upload-time = "2026-05-06T15:10:39.569Z" }, + { url = "https://files.pythonhosted.org/packages/67/bd/2775ff28bfe883b9aa1ff348300542eb2ef1ee18d8ae0e3a49846817a865/orjson-3.11.9-cp313-cp313-win_amd64.whl", hash = "sha256:051b102c93b4f634e89f3866b07b9a9a98915ada541f4ec30f177067b2694979", size = 127086, upload-time = "2026-05-06T15:10:41.262Z" }, + { url = "https://files.pythonhosted.org/packages/91/2b/d26799e580939e32a7da9a39531bc9e58e15ca32ffaa6a8cb3e9bb0d22cd/orjson-3.11.9-cp313-cp313-win_arm64.whl", hash = "sha256:cce9127885941bd28f080cecf1f1d288336b7e0d812c345b08be88b572796254", size = 126696, upload-time = "2026-05-06T15:10:42.651Z" }, + { url = "https://files.pythonhosted.org/packages/8e/eb/5da01e356015aee6ecfa1187ced87aef51364e306f5e695dd52719bf0e78/orjson-3.11.9-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b6ef1979adc4bc243523f1a2ba91418030a8e29b0a99cbe7e0e2d6807d4dce6e", size = 228465, upload-time = "2026-05-06T15:10:44.097Z" }, + { url = "https://files.pythonhosted.org/packages/64/62/3e0e0c14c957133bcd855395c62b55ed4e3b0af23ffea11b032cb1dcbdb1/orjson-3.11.9-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:f36b7f32c7c0db4a719f1fc5824db4a9c6f8bd1a354debb91faf26ebf3a4c71e", size = 128364, upload-time = "2026-05-06T15:10:45.839Z" }, + { url = "https://files.pythonhosted.org/packages/5a/5a/07d8aa117211a8ed7630bda80c8c0b14d04e0f8dcf99bcf49656e4a710eb/orjson-3.11.9-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08f4d8ebb44925c794e535b2bebc507cebf32209df81de22ae285fb0d8d66de0", size = 132063, upload-time = "2026-05-06T15:10:47.267Z" }, + { url = "https://files.pythonhosted.org/packages/d6/ec/4acaf21483e18aa945be74a474c74b434f284b549f275a0a39b9f98956e9/orjson-3.11.9-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6cc7923789694fd58f001cbcac7e47abc13af4d560ebbfcf3b41a8b1a0748124", size = 122356, upload-time = "2026-05-06T15:10:48.765Z" }, + { url = "https://files.pythonhosted.org/packages/13/d8/5f0555e7638801323b7a75850f92e7dfa891bc84fe27a1ba4449170d1200/orjson-3.11.9-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea5c46eb2d3af39e806b986f4b09d5c2706a1f5afde3cbf7544ce6616127173c", size = 129592, upload-time = "2026-05-06T15:10:50.13Z" }, + { url = "https://files.pythonhosted.org/packages/b6/30/ed9860412a3603ceb3c5955bfd72d28b9d0e7ba6ed81add14f83d7114236/orjson-3.11.9-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f5d89a2ed90731df3be64bab0aa44f78bff39fdc9d71c291f4a8023aa46425b7", size = 140491, upload-time = "2026-05-06T15:10:51.582Z" }, + { url = "https://files.pythonhosted.org/packages/d0/17/adc514dea7ac7c505527febf884934b815d34f0c7b8693c1a8b39c5c4a57/orjson-3.11.9-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:25e4aed0312d292c09f61af25bba34e0b2c88546041472b09088c39a4d828af1", size = 127309, upload-time = "2026-05-06T15:10:53.329Z" }, + { url = "https://files.pythonhosted.org/packages/76/3e/c0b690253f0b82d86e99949af13533363acfb5432ecb5d53dd5b3bce9c34/orjson-3.11.9-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaea64f3f467d22e70eeed68bdccb3bc4f83f650446c4a03c59f2cba28a108db", size = 134030, upload-time = "2026-05-06T15:10:54.988Z" }, + { url = "https://files.pythonhosted.org/packages/c1/7a/bc82a0bb25e9faaf92dc4d9ef002732efc09737706af83e346788641d4a7/orjson-3.11.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a028425d1b440c5d92a6be1e1a020739dfe67ea87d96c6dbe828c1b30041728b", size = 141482, upload-time = "2026-05-06T15:10:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/01/55/e69188b939f77d5d32a9833745ace31ea5ccae3ab613a1ec185d3cd2c4fb/orjson-3.11.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5b192c6cf397e4455b11523c5cf2b18ed084c1bbd61b6c0926344d2129481972", size = 415178, upload-time = "2026-05-06T15:10:58.446Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1a/b8a5a7ac527e80b9cb11d51e3f6689b709279183264b9ec5c7bc680bb8b5/orjson-3.11.9-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea407d4ccf5891d667d045fecae97a7a1e5e87b3b97f97ae1803c2e741130be0", size = 148089, upload-time = "2026-05-06T15:11:00.441Z" }, + { url = "https://files.pythonhosted.org/packages/97/4e/00503f64204bf859b37213a63927028f30fb6268cd8677fb0a5ad48155e1/orjson-3.11.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f63aaf97afd9f6dec5b1a68e1b8da12bfccb4cb9a9a65c3e0b6c847849e7586", size = 136921, upload-time = "2026-05-06T15:11:02.176Z" }, + { url = "https://files.pythonhosted.org/packages/0d/ba/a23b82a0a8d0ed7bed4e5f5035aae751cad4ff6a1e8d2ecd14d8860f5929/orjson-3.11.9-cp314-cp314-win32.whl", hash = "sha256:e30ab17845bb9fa54ccf67fa4f9f5282652d54faa6d17452f47d0f369d038673", size = 131638, upload-time = "2026-05-06T15:11:03.696Z" }, + { url = "https://files.pythonhosted.org/packages/f3/c3/0c6798456bade745c75c452342dabacce5798196483e77e643be1f53877d/orjson-3.11.9-cp314-cp314-win_amd64.whl", hash = "sha256:32ef5f4283a3be81913947d19608eacb7c6608026851123790cd9cc8982af34b", size = 127078, upload-time = "2026-05-06T15:11:05.123Z" }, + { url = "https://files.pythonhosted.org/packages/16/21/5a3f1e8913103b703a436a5664238e5b965ec392b555fe68943ea3691e6b/orjson-3.11.9-cp314-cp314-win_arm64.whl", hash = "sha256:eebdbdeef0094e4f5aefa20dcd4eb2368ab5e7a3b4edea27f1e7b2892e009cf9", size = 126687, upload-time = "2026-05-06T15:11:06.602Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -692,40 +812,63 @@ wheels = [ ] [[package]] -name = "pluggy" -version = "1.6.0" +name = "pathvalidate" +version = "3.3.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/2a/52a8da6fe965dea6192eb716b357558e103aea0a1e9a8352ad575a8406ca/pathvalidate-3.3.1.tar.gz", hash = "sha256:b18c07212bfead624345bb8e1d6141cdcf15a39736994ea0b94035ad2b1ba177", size = 63262, upload-time = "2025-06-15T09:07:20.736Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, + { url = "https://files.pythonhosted.org/packages/9a/70/875f4a23bfc4731703a5835487d0d2fb999031bd415e7d17c0ae615c18b7/pathvalidate-3.3.1-py3-none-any.whl", hash = "sha256:5263baab691f8e1af96092fa5137ee17df5bdfbd6cff1fcac4d6ef4bc2e1735f", size = 24305, upload-time = "2025-06-15T09:07:19.117Z" }, ] [[package]] -name = "polars" -version = "1.41.2" +name = "pendulum" +version = "3.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "polars-runtime-32" }, + { name = "python-dateutil" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/72/9a51afa0a822b09e286c4cb827ed7b00bc818dac7bd11a5f161e493a217d/pendulum-3.2.0.tar.gz", hash = "sha256:e80feda2d10fa3ff8b1526715f7d33dcb7e08494b3088f2c8a3ac92d4a4331ce", size = 86912, upload-time = "2026-01-30T11:22:24.093Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/8c/400c8b8dbd7524424f3d9902ded64741e82e5e321d1aabbd68ade89e71cf/pendulum-3.2.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:addb0512f919fe5b70c8ee534ee71c775630d3efe567ea5763d92acff857cfc3", size = 337820, upload-time = "2026-01-30T11:21:24.305Z" }, + { url = "https://files.pythonhosted.org/packages/59/38/7c16f26cc55d9206d71da294ce6857d0da381e26bc9e0c2a069424c2b173/pendulum-3.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3aaa50342dc174acebdc21089315012e63789353957b39ac83cac9f9fc8d1075", size = 327551, upload-time = "2026-01-30T11:21:25.747Z" }, + { url = "https://files.pythonhosted.org/packages/0b/cd/f36ec5d56d55104232380fdbf84ff53cc05607574af3cbdc8a43991ac8a7/pendulum-3.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:927e9c9ab52ff68e71b76dd410e5f1cd78f5ea6e7f0a9f5eb549aea16a4d5354", size = 339894, upload-time = "2026-01-30T11:21:27.229Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4e/b9a1e546519c3a92d5bc17787cea925e06a20def2ae344fa136d2fc40338/pendulum-3.2.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:249d18f5543c9f43aba3bd77b34864ec8cf6f64edbead405f442e23c94fce63d", size = 373766, upload-time = "2026-01-30T11:21:28.642Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a6/6471ab87ae2260594501f071586a765fc894817043b7d2d4b04e2eff4f31/pendulum-3.2.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c644cc15eec5fb02291f0f193195156780fd5a0affd7a349592403826d1a35e", size = 379837, upload-time = "2026-01-30T11:21:30.637Z" }, + { url = "https://files.pythonhosted.org/packages/0d/79/0ba0c14e862388f7b822626e6e989163c23bebe7f96de5ec4b207cbe7c3d/pendulum-3.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:063ab61af953bb56ad5bc8e131fd0431c915ed766d90ccecd7549c8090b51004", size = 348904, upload-time = "2026-01-30T11:21:32.436Z" }, + { url = "https://files.pythonhosted.org/packages/17/34/df922c7c0b12719589d4954bfa5bdca9e02bcde220f5c5c1838a87118960/pendulum-3.2.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:26a3ae26c9dd70a4256f1c2f51addc43641813574c0db6ce5664f9861cd93621", size = 517173, upload-time = "2026-01-30T11:21:34.428Z" }, + { url = "https://files.pythonhosted.org/packages/87/ec/3b9e061eeee97b72a47c1434ee03f6d85f0284d9285d92b12b0fff2d19ac/pendulum-3.2.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:2b10d91dc00f424444a42f47c69e6b3bfd79376f330179dc06bc342184b35f9a", size = 561744, upload-time = "2026-01-30T11:21:35.861Z" }, + { url = "https://files.pythonhosted.org/packages/fd/7e/f12fdb6070b7975c1fcfa5685dbe4ab73c788878a71f4d1d7e3c87979e37/pendulum-3.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:63070ff03e30a57b16c8e793ee27da8dac4123c1d6e0cf74c460ce9ee8a64aa4", size = 258746, upload-time = "2026-01-30T11:21:37.782Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b8/5abd872056357f069ae34a9b24a75ac58e79092d16201d779a8dd31386bb/pendulum-3.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:c8dde63e2796b62070a49ce813ce200aba9186130307f04ec78affcf6c2e8122", size = 253028, upload-time = "2026-01-30T11:21:39.381Z" }, + { url = "https://files.pythonhosted.org/packages/82/99/5b9cc823862450910bcb2c7cdc6884c0939b268639146d30e4a4f55eb1f1/pendulum-3.2.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c17ac069e88c5a1e930a5ae0ef17357a14b9cc5a28abadda74eaa8106d241c8e", size = 338281, upload-time = "2026-01-30T11:21:40.812Z" }, + { url = "https://files.pythonhosted.org/packages/cd/3a/64a35260f6ac36c0ad50eeb5f1a465b98b0d7603f79a5c2077c41326d639/pendulum-3.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e1fbb540edecb21f8244aebfb05a1f2333ddc6c7819378c099d4a61cc91ae93c", size = 328030, upload-time = "2026-01-30T11:21:42.778Z" }, + { url = "https://files.pythonhosted.org/packages/da/6b/1140e09310035a2afb05bb90a2b8fbda9d3222e03b92de9533123afe6b65/pendulum-3.2.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8c67fb9a1fe8fc1adae2cc01b0c292b268c12475b4609ff4aed71c9dd367b4d", size = 340206, upload-time = "2026-01-30T11:21:44.148Z" }, + { url = "https://files.pythonhosted.org/packages/52/4a/a493de56cbc24a64b21ac6ba98513a9ec5c67daa3dba325e39a8e53f30d8/pendulum-3.2.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:baa9a66c980defda6cfe1275103a94b22e90d83ebd7a84cc961cee6cbd25a244", size = 373976, upload-time = "2026-01-30T11:21:45.56Z" }, + { url = "https://files.pythonhosted.org/packages/3c/4c/f083c4fd1a161d4ab218680cc906338c541497b3098373f2241f58c429cb/pendulum-3.2.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef8f783fa7a14973b0596d8af2a5b2d90858a55030e9b4c6885eb4284b88314f", size = 380075, upload-time = "2026-01-30T11:21:46.959Z" }, + { url = "https://files.pythonhosted.org/packages/57/b6/333a0fcb33bf15eb879a46a11ce6300c1698a141e689665fe430783ff8d6/pendulum-3.2.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7d2e9bfb065727d8676e7ada3793b47a24349500a5e9637404355e482c822be", size = 349026, upload-time = "2026-01-30T11:21:48.271Z" }, + { url = "https://files.pythonhosted.org/packages/43/1a/dfb526ec0cba1e7cd6a5e4f4dd64a6ada7428d1449c54b15f7b295f6e122/pendulum-3.2.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:55d7ba6bb74171c3ee409bf30076ee3a259a3c2bb147ac87ebb76aaa3cf5d3a2", size = 517395, upload-time = "2026-01-30T11:21:49.643Z" }, + { url = "https://files.pythonhosted.org/packages/c9/37/b4f2b5f1200351c4869b8b46ad5c21019e3dbe0417f5867ae969fad7b5fe/pendulum-3.2.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:a50d8cf42f06d3d8c3f8bb2a7ac47fa93b5145e69de6a7209be6a47afdd9cf76", size = 561926, upload-time = "2026-01-30T11:21:51.698Z" }, + { url = "https://files.pythonhosted.org/packages/a0/9e/567376582da58f5fe8e4f579db2bcfbf243cf619a5825bdf1023ad1436b3/pendulum-3.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:e5bbb92b155cd5018b3cf70ee49ed3b9c94398caaaa7ed97fe41e5bb5a968418", size = 258817, upload-time = "2026-01-30T11:21:53.074Z" }, + { url = "https://files.pythonhosted.org/packages/95/67/dfffd7eb50d67fa821cd4d92cf71575ead6162930202bc40dfcedf78c38c/pendulum-3.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:d53134418e04335c3029a32e9341cccc9b085a28744fb5ee4e6a8f5039363b1a", size = 253292, upload-time = "2026-01-30T11:21:54.484Z" }, + { url = "https://files.pythonhosted.org/packages/02/fb/d65db067a67df7252f18b0cb7420dda84078b9e8bfb375215469c14a50be/pendulum-3.2.0-py3-none-any.whl", hash = "sha256:f3a9c18a89b4d9ef39c5fa6a78722aaff8d5be2597c129a3b16b9f40a561acf3", size = 114111, upload-time = "2026-01-30T11:22:22.361Z" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ff/f9/aeda46259b0669247a160315d2d51269de9504b9dd2f70acadbcb22f46b7/polars-1.41.2.tar.gz", hash = "sha256:256d6731162371b77f3f29a55eacb8c0fc740ddb1a293a01d2ef5b5393c5c708", size = 737996, upload-time = "2026-05-29T17:39:15.604Z" } + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1f/22/28f62d24f7db56ac4343588f9362d49b7b4177e55ac47a466fe696b0099b/polars-1.41.2-py3-none-any.whl", hash = "sha256:23ce9a2910b6e3e8d4258770bf44aa17170958df7af6e85feedf4458a04d8d29", size = 833445, upload-time = "2026-05-29T17:37:05.576Z" }, + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] [[package]] -name = "polars-runtime-32" -version = "1.41.2" +name = "ply" +version = "3.11" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f9/56/54e3ea0e9b64f327179049e4742241cc6b1d3e8fa414b05a057dd26df367/polars_runtime_32-1.41.2.tar.gz", hash = "sha256:7af09ec1ab053da2c9669e8d15f809a4083a29be05db57111688b8051062af56", size = 2989474, upload-time = "2026-05-29T17:39:17.257Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/69/882ee5c9d017149285cab114ebeab373308ef0f874fcdac9beb90e0ac4da/ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3", size = 159130, upload-time = "2018-02-15T19:01:31.097Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d6/9b/fe72a3811c0357cdb06c67bdc7695fa1623ad47948fc523195f5ac31037f/polars_runtime_32-1.41.2-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:95a08346dac337357cdb825c8076df7d36da54c4caa59a5cb41d0a30691c5edd", size = 52265283, upload-time = "2026-05-29T17:37:09.407Z" }, - { url = "https://files.pythonhosted.org/packages/0a/93/fab9da803fd80d9e83ef88c20932f637a10bc611b20415fc322eec84bc44/polars_runtime_32-1.41.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:dedfaeec2c7f995298da7319dd9431d662e5dd1d0ec51b1459df4a0234ceff52", size = 46571222, upload-time = "2026-05-29T17:37:13.698Z" }, - { url = "https://files.pythonhosted.org/packages/c8/2a/8843f34a8ac57acd058a39b87b03b580dd352a490e9dae0415e02033bdd4/polars_runtime_32-1.41.2-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18eea22c5cc34e27f8a60950458ad81e6a9ea75e89363ca1367e14e7e7f781fc", size = 50409372, upload-time = "2026-05-29T17:37:17.875Z" }, - { url = "https://files.pythonhosted.org/packages/6c/c6/92b352fe88cf51bd0a19fb99e1c0cbe46aa26c14dcf7995b89869cd932ae/polars_runtime_32-1.41.2-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2630540dfdfb0f36f9b04a07c7c2e3f50bf2ad384113263c1c812007ee9141e0", size = 56405484, upload-time = "2026-05-29T17:37:22.684Z" }, - { url = "https://files.pythonhosted.org/packages/74/c4/bae3174c3b02f6b441d2e58594387abcd509f67a098f682a83b195f08966/polars_runtime_32-1.41.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:20e969e08f9b137e233c04cc04de73d9795f89eb77d34854e40a025965a43763", size = 50603512, upload-time = "2026-05-29T17:37:27.422Z" }, - { url = "https://files.pythonhosted.org/packages/f4/ed/f2d26ae02d92c2689056838ed59e2a626326ad23c2831d58637d25f6c82a/polars_runtime_32-1.41.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e7016a3deb641b64a31447abbbee0f34bd020a6a9ae34ee6b743837def15e2a4", size = 54328561, upload-time = "2026-05-29T17:37:32.587Z" }, - { url = "https://files.pythonhosted.org/packages/9b/c4/9c3831cc885dc7769e59abf8f583821a5fb4403fd0e4eba0ccc6d47a3d4b/polars_runtime_32-1.41.2-cp310-abi3-win_amd64.whl", hash = "sha256:1e5e5377c315e0dcafdfb2a31adc546abbaeb3f9cb1864e6536523d2af473265", size = 51978643, upload-time = "2026-05-29T17:37:37.443Z" }, - { url = "https://files.pythonhosted.org/packages/cd/c6/79e9f3f270270d7ed5575d92b7bfef49f01abd9275447161275b23b553a8/polars_runtime_32-1.41.2-cp310-abi3-win_arm64.whl", hash = "sha256:843d96f69d18eca53429c1198e58891db7f18111f83b9c419bb45ad9d73eaed5", size = 46006901, upload-time = "2026-05-29T17:37:42.522Z" }, + { url = "https://files.pythonhosted.org/packages/a3/58/35da89ee790598a0700ea49b2a66594140f44dec458c07e8e3d4979137fc/ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce", size = 49567, upload-time = "2018-02-15T19:01:27.172Z" }, ] [[package]] @@ -918,6 +1061,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126", size = 510141, upload-time = "2026-05-04T01:35:27.408Z" }, ] +[[package]] +name = "pywin32" +version = "312" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/41/12fbfd7f36ed2146d8bc9de96c2741296bf0d490b98508496cff322e274c/pywin32-312-cp313-cp313-win32.whl", hash = "sha256:7a27df850933d16a8eabfbaeb73d52b273e2da667f80d70b01a89d1f6828d02c", size = 6370184, upload-time = "2026-06-04T07:49:36.253Z" }, + { url = "https://files.pythonhosted.org/packages/ba/db/36a78e3403099d31d9746d13fdcde5accc43c1155f375a34d15983a479a7/pywin32-312-cp313-cp313-win_amd64.whl", hash = "sha256:c53e878d15a1c44788082bfe712a905433473aa38f86375b7cf8b45e3acbaaf9", size = 6914298, upload-time = "2026-06-04T07:49:38.876Z" }, + { url = "https://files.pythonhosted.org/packages/84/37/c1697194092b76de9ed47ca124323f02c57ffc8a45c06f88a3d5acaf01eb/pywin32-312-cp313-cp313-win_arm64.whl", hash = "sha256:59aba5d5940842075343a5ddc6b11f1cdf0d1567fe745290359dfbcc7c2eb831", size = 6727640, upload-time = "2026-06-04T07:49:41.083Z" }, + { url = "https://files.pythonhosted.org/packages/fc/2b/1f3cded5822fd49c02f40544cbb5f58c7cfd6b1694869fd476cb6170ee97/pywin32-312-cp314-cp314-win32.whl", hash = "sha256:a77a90fbb6881238d2ca9c6fd797b25817f3768fe78d214a90137ff055a75f5b", size = 6468928, upload-time = "2026-06-04T07:49:43.188Z" }, + { url = "https://files.pythonhosted.org/packages/21/82/3bf86d2e2808902013132e1ce905a7da0da53790f3836c64bf44d55e24f3/pywin32-312-cp314-cp314-win_amd64.whl", hash = "sha256:a4dd3a848290ef724347b19f301045831d8e802fa4464f491b98b1e0a081432e", size = 7024157, upload-time = "2026-06-04T07:49:45.34Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0e/73f6d6800b4f27655abd9e9f6aaeaefcddb2b946e4674efa2bab184a7f7b/pywin32-312-cp314-cp314-win_arm64.whl", hash = "sha256:9fce94568364e0155e6dfb781ac5d95903be8baf28670632beab1b523f300daa", size = 6839598, upload-time = "2026-06-04T07:49:47.613Z" }, + { url = "https://files.pythonhosted.org/packages/eb/61/caa39686032d2ebdd04ff0ab5cbe163126c0066d98e00c9018646e42393b/pywin32-312-cp315-cp315-win32.whl", hash = "sha256:5c1fbe4a937a73ae9297384a3da38518cbc694c68ad8a809b2e19acd350f03ed", size = 6471159, upload-time = "2026-06-04T07:49:50.035Z" }, + { url = "https://files.pythonhosted.org/packages/0f/cd/7e1de64a4a6f69c04214169657ccab0d93a670ea50e35eb8f489d7378249/pywin32-312-cp315-cp315-win_amd64.whl", hash = "sha256:c2f03a0f73f804a13c2735b99392b0cd426bb4f2c4d0178e5ac966a0f21618d5", size = 7025293, upload-time = "2026-06-04T07:49:54.857Z" }, + { url = "https://files.pythonhosted.org/packages/23/ed/4532e9388e65fa16b46776ef47ad631a64eda1631884488af707666350ed/pywin32-312-cp315-cp315-win_arm64.whl", hash = "sha256:a8597d28f267b39074aef51fa593530082b39cbe5a074226096857b1fed2dfb9", size = 6840337, upload-time = "2026-06-04T07:49:57.531Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -972,9 +1131,8 @@ name = "repolytics" version = "0.1.0" source = { editable = "." } dependencies = [ + { name = "dlt", extra = ["duckdb"] }, { name = "duckdb" }, - { name = "httpx" }, - { name = "polars" }, { name = "pydantic-settings" }, ] @@ -991,9 +1149,8 @@ dev = [ [package.metadata] requires-dist = [ + { name = "dlt", extras = ["duckdb"], specifier = ">=1.28.0" }, { name = "duckdb", specifier = ">=1.5.3" }, - { name = "httpx", specifier = ">=0.28.1" }, - { name = "polars", specifier = ">=1.41.2" }, { name = "pydantic-settings", specifier = ">=2.14.1" }, ] @@ -1023,6 +1180,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[[package]] +name = "requirements-parser" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/96/fb6dbfebb524d5601d359a47c78fe7ba1eef90fc4096404aa60c9a906fbb/requirements_parser-0.13.0.tar.gz", hash = "sha256:0843119ca2cb2331de4eb31b10d70462e39ace698fd660a915c247d2301a4418", size = 22630, upload-time = "2025-05-21T13:42:05.464Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/60/50fbb6ffb35f733654466f1a90d162bcbea358adc3b0871339254fbc37b2/requirements_parser-0.13.0-py3-none-any.whl", hash = "sha256:2b3173faecf19ec5501971b7222d38f04cb45bb9d87d0ad629ca71e2e62ded14", size = 14782, upload-time = "2025-05-21T13:42:04.007Z" }, +] + +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + +[[package]] +name = "rich-argparse" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/e5/1064c43203a357d668cd42435f7a15fe6af51512d85b2104fecb937aa861/rich_argparse-1.8.0.tar.gz", hash = "sha256:679df3d832fa94ad6e4bdb07ded088cd7ea2dddc58ae9b2b46346a40b06cbc0c", size = 38940, upload-time = "2026-05-01T15:18:43.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/35/1cceccc5fcb50fa2ed53e2aa278cd032f3902682a73e763fb1ac3be8e6fa/rich_argparse-1.8.0-py3-none-any.whl", hash = "sha256:d2a3ce7854654e2253c578763ab0a32f05016f23a55fadba7b9a91b6c0e92142", size = 25616, upload-time = "2026-05-01T15:18:42.395Z" }, +] + [[package]] name = "rpds-py" version = "2026.5.1" @@ -1143,6 +1337,66 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2d/c7/c53e8dbff9c9dc4b7928773421ae294a5d28fcb8dcda1a089579d3a7e510/ruff-0.15.17-py3-none-win_arm64.whl", hash = "sha256:f3be1fbb34bcdfd146240d8fb92a709d4c2c8191348580a3c044ec60fa0b4456", size = 11355275, upload-time = "2026-06-11T17:54:43.635Z" }, ] +[[package]] +name = "semver" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/d1/d3159231aec234a59dd7d601e9dd9fe96f3afff15efd33c1070019b26132/semver-3.0.4.tar.gz", hash = "sha256:afc7d8c584a5ed0a11033af086e8af226a9c0b206f313e0301f8dd7b6b589602", size = 269730, upload-time = "2025-01-24T13:19:27.617Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl", hash = "sha256:9c824d87ba7f7ab4a1890799cec8596f15c1241cb473404ea1cb0c55e4b04746", size = 17912, upload-time = "2025-01-24T13:19:24.949Z" }, +] + +[[package]] +name = "setuptools" +version = "82.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4f/db/cfac1baf10650ab4d1c111714410d2fbb77ac5a616db26775db562c8fab2/setuptools-82.0.1.tar.gz", hash = "sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9", size = 1152316, upload-time = "2026-03-09T12:47:17.221Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", hash = "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb", size = 1006223, upload-time = "2026-03-09T12:47:15.026Z" }, +] + +[[package]] +name = "simplejson" +version = "4.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0e/2a/54837395a3487c725669428d513293612a48d82b95a0642c936932e5d898/simplejson-4.1.1.tar.gz", hash = "sha256:c08eb9f7a90f77ae470e19a07472e9a79ebc0d1c2315d86a72767665bd5ba79f", size = 118860, upload-time = "2026-04-24T19:24:59.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/a9/47b445eeb559c9593453a0648e0fd6d08e8adff64dd5e5ced66726da8a09/simplejson-4.1.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dff52fc7af272e84fc21cc5a06c927c823ca6ae00af14f3b0d7707b42775ed98", size = 113160, upload-time = "2026-04-24T19:23:26.033Z" }, + { url = "https://files.pythonhosted.org/packages/4c/65/cb72db31523c164dea5dc55b02dad065a40c478856bc7534b279d2b51906/simplejson-4.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:971aed0647ad6e840a3943bec812fcda5f2d26a5497a4981d1fb49aa4f9a396c", size = 91521, upload-time = "2026-04-24T19:23:27.572Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e5/54cb7c50ad5fdc1e0a86b7df4b135c2cbd5c4623605aa94466659098e8da/simplejson-4.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:249e2e220aa6d9b9d936bde84eb7bf79d5b6c5a8273c6e411f8b1635a9073f2d", size = 91407, upload-time = "2026-04-24T19:23:28.991Z" }, + { url = "https://files.pythonhosted.org/packages/38/2e/21a3ede87f0bf82d6c7bcb90480d50a6490eb974c6ab20881188e440957c/simplejson-4.1.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8e5cdd6a5d52299f345c15ab5678cc4249e24f383f361d986afbc3c7072a6b6b", size = 192451, upload-time = "2026-04-24T19:23:30.56Z" }, + { url = "https://files.pythonhosted.org/packages/59/df/9903edd3102bf0b5984edfcb90c88612330996efa3b4fbf8a971d6e17839/simplejson-4.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:642cec364e0676e2d5a73fa4d31d0c7c55886997caa2fde24e8292ca44d32728", size = 189015, upload-time = "2026-04-24T19:23:32.647Z" }, + { url = "https://files.pythonhosted.org/packages/98/cd/33230927a780e1398b857e3944abb914556994d252b1d765ae40d112cb25/simplejson-4.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:76fe296ca1df23d290033f10aaacf534fd1b3e3007e7f9ff8aa68b21413aaa78", size = 196658, upload-time = "2026-04-24T19:23:34.563Z" }, + { url = "https://files.pythonhosted.org/packages/cd/84/2c5a7444eb53e9a86d3738299bffddd9f53aeed799ded2f45368221fdb19/simplejson-4.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8f0ad25b7dc4e0fb23858355819f2e994f1a5badcdcde8737eac7921c2f1ed2a", size = 185967, upload-time = "2026-04-24T19:23:36.191Z" }, + { url = "https://files.pythonhosted.org/packages/d3/68/454378e06d059cd412a7ed5d87fb6d29fd5b60f13a4d89fc1f764ff434df/simplejson-4.1.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a59ebd0533f03fd06ff0c42ba0f02d93cbcdd7944922bf3b93911327a95b901f", size = 193940, upload-time = "2026-04-24T19:23:38.151Z" }, + { url = "https://files.pythonhosted.org/packages/d5/d5/a15bf915f623a2c5a079d6e3be8256fdb8ef06f110669493a09b9d6933e0/simplejson-4.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bccbf4419676b517939852e5aeff2af6aee4dc046881c67a1581fa6f1cb01abd", size = 189795, upload-time = "2026-04-24T19:23:40.139Z" }, + { url = "https://files.pythonhosted.org/packages/d2/c9/37212ae7dc4b607f0978c408e8633f05c810884e054c33113184c6c2c8a2/simplejson-4.1.1-cp313-cp313-win32.whl", hash = "sha256:6c845363eb5fd166fb7c72243da38f4fcfde666ede7fdf2cc6fd7762894626f7", size = 88773, upload-time = "2026-04-24T19:23:41.754Z" }, + { url = "https://files.pythonhosted.org/packages/fe/a5/c7a0a47883a9015b54c9d8a4b62f2aba17bd4335b1787b9b8a0fc2fa6d52/simplejson-4.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:104d8324c34f25b4b90800bc5fa363780cbc3d8496aef061cba7ce1af9162270", size = 90888, upload-time = "2026-04-24T19:23:43.11Z" }, + { url = "https://files.pythonhosted.org/packages/d3/18/4a118a6a92eb33bb08c8e2fe7ec85cb96f0673491bb2b829930831ee4fbe/simplejson-4.1.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:ed7473602b6625de793b6acba49aa949f144a475f538792067e4cf2fda2071f5", size = 110492, upload-time = "2026-04-24T19:23:44.957Z" }, + { url = "https://files.pythonhosted.org/packages/07/f4/84d160e9fa8cada1e0a9381cae4fa81eecd573577a5b34366d8ced59bdf7/simplejson-4.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:225c9caa324c5b554d009fb9cac22aee7711e71bd96f487938c659af467e828e", size = 90152, upload-time = "2026-04-24T19:23:46.355Z" }, + { url = "https://files.pythonhosted.org/packages/68/31/9a5432c433a7671107182cdc9a20ea78a70f99c4e5334aa54b6d4d0d79ed/simplejson-4.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:95407269340c7f22f09776ea7b717a52cf56cfcf119b5e45f66faa4a26445bea", size = 90115, upload-time = "2026-04-24T19:23:47.743Z" }, + { url = "https://files.pythonhosted.org/packages/78/91/3635cdb13318cb0a328abaa69e2b91251caad39d6779aa308098f341f6cb/simplejson-4.1.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3851658d642c1184d2023f0e6c9ce44a21eb1629e74e7c84ef956b128841fe12", size = 184036, upload-time = "2026-04-24T19:23:49.472Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ba/149b6ec5393f6849d98c59cadba888b710a8ef4b805ab91e11a566960d40/simplejson-4.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:95a3bb0f78e85f4937f99092239f2011ce06f0f2d803df5c299cc05abbeae008", size = 180543, upload-time = "2026-04-24T19:23:51.023Z" }, + { url = "https://files.pythonhosted.org/packages/df/7c/a5d968d0b527a748b667e62bea94309ccbcb1e2b108e8f0cf8547efaa12b/simplejson-4.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bbfdaa7c0603f75b7b14b211b7f2be44696d4e26833ad2d91d5c87bf5fb9a920", size = 188725, upload-time = "2026-04-24T19:23:52.995Z" }, + { url = "https://files.pythonhosted.org/packages/db/e3/6a8d11181d587ef00e2db9112357e6832111e56dd56b01b5c11758a1965d/simplejson-4.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:39e3c584071dced8c21b4689f0254303521daeb9b5bc1f4289755d71fa3cb0d3", size = 177492, upload-time = "2026-04-24T19:23:54.581Z" }, + { url = "https://files.pythonhosted.org/packages/67/e3/8b0eb8b06e8198cfbd1270487da163d0093df05cc4f557350cd65e2f7e79/simplejson-4.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:036a27bd0469b9d79557cbddb392969f876cd7f278cfbd0fba81534927a06575", size = 185281, upload-time = "2026-04-24T19:23:56.13Z" }, + { url = "https://files.pythonhosted.org/packages/dc/5f/64990f07ec9e2cb1a814c674e2e21b5693207f74ac70eb72151b847ea4e6/simplejson-4.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b70bfd2f67f3351baba08aa3ae9233c83f21fd95ae5e6b3d0ecb8c647929112f", size = 181848, upload-time = "2026-04-24T19:23:57.92Z" }, + { url = "https://files.pythonhosted.org/packages/61/a5/bbc1bc0447f339f79f99ab8c37f7f037cb2f1f93af75d6a4d553096bb0c3/simplejson-4.1.1-cp314-cp314-win32.whl", hash = "sha256:37233c72ce88d06acb92747347742b3c07871eba6789f060c179c9302dde8efe", size = 88761, upload-time = "2026-04-24T19:23:59.397Z" }, + { url = "https://files.pythonhosted.org/packages/18/72/ec1b5cbdcb140c132e6c7bdf99bd73e4f675439e77126c88f472fcffa09c/simplejson-4.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:cc0442dea71cd9cbf30a0b8b9929ab5aa6c02c0443a3d977351e6ec5bada4388", size = 91018, upload-time = "2026-04-24T19:24:00.85Z" }, + { url = "https://files.pythonhosted.org/packages/3d/97/4fa437f68ff72219bac3bf3d050de9c6265691f3a170e16954bd69d7cddd/simplejson-4.1.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:c996a4d38290c515af347740659ce095b425449c164a5c9fa3977caa6eff5dbe", size = 113919, upload-time = "2026-04-24T19:24:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/c2/83/59de041d09eb4a9577f7015d7263c32095dfb7fde49717dff62145d89809/simplejson-4.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c65c763fb20d7ca113c1c14dce2fc04a0fc3a57aceff533d6fdac707c7bffb40", size = 91904, upload-time = "2026-04-24T19:24:03.812Z" }, + { url = "https://files.pythonhosted.org/packages/03/8e/46bb345d540f6eb31427d984a4e518cdb182d0621814fee4fee045e8815b/simplejson-4.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0da5c9f57206ee7ef280ff7f1d924937b0a64f9a271a5ef371a2ecdbebba7421", size = 91752, upload-time = "2026-04-24T19:24:05.622Z" }, + { url = "https://files.pythonhosted.org/packages/83/e2/1b2ce97f068835eb3d253c116a4df7a3f436b7bf2fb5ff1ba29287e8b0ec/simplejson-4.1.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ea3426e786425d10e9e82f8a6eda74a7d6eb10d99165ac3d0d3bbcb65c0ea343", size = 214021, upload-time = "2026-04-24T19:24:07.447Z" }, + { url = "https://files.pythonhosted.org/packages/48/70/d93e556df6a0786298644a7c08304fcbeddc248325f23f38acbebeb21165/simplejson-4.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d75cea7a1025edd7e439b2966b3d977c45b5b899e2adaf422811b3ac702ed9fb", size = 213530, upload-time = "2026-04-24T19:24:09.289Z" }, + { url = "https://files.pythonhosted.org/packages/1b/a5/c93bf305b9f00d7259e09e713d60e75bd0f7f53da970f716ab90491770e7/simplejson-4.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63c2ada8e58f266491f19eed2eeeb7c25c6141e52f8f9e820f6bb94156cf8dbc", size = 218282, upload-time = "2026-04-24T19:24:10.991Z" }, + { url = "https://files.pythonhosted.org/packages/0c/20/a9b5d2e27ec44b069ee251bd55544fc76929a067107b1050001566ba86f3/simplejson-4.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d1fffb56305c5b475ee746cf9e04f97423ba5aaacd292dc1255bd75b1d3b124b", size = 209249, upload-time = "2026-04-24T19:24:12.662Z" }, + { url = "https://files.pythonhosted.org/packages/97/e4/e06ee682ed5df67592181f5ecb062e35878967e27f5b6e087237d4548d95/simplejson-4.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:a6525ec733f43d0541206cffa64fd2aad5a7ae3eb76566aff49cd4db6382209a", size = 213963, upload-time = "2026-04-24T19:24:14.302Z" }, + { url = "https://files.pythonhosted.org/packages/9c/9f/1e160e4cd8cdbf062bf6a454cdf814dc7a48eb47e566fdb8f80ccb202605/simplejson-4.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:861e393260508efa64d8805a8e49c416c3484907e3f146ce966c69552b49b9a3", size = 210474, upload-time = "2026-04-24T19:24:15.917Z" }, + { url = "https://files.pythonhosted.org/packages/7a/e6/cecd913df322df5bbe7ebb8ba39e0708e505a165553900da8a7761026d6f/simplejson-4.1.1-cp314-cp314t-win32.whl", hash = "sha256:d083b89d30948a751d3d97476c2ed91e4caaa24a1a1459bdbadb8876242c71fe", size = 91134, upload-time = "2026-04-24T19:24:17.635Z" }, + { url = "https://files.pythonhosted.org/packages/97/73/f540dde99cc1d393bd062ab3b5735b777561a5d8f8a5f2e241164444d77a/simplejson-4.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:4cbb299d0528ec0447fe366d8c9641860e28f997a62730690fef905f1f41046e", size = 94467, upload-time = "2026-04-24T19:24:19.109Z" }, + { url = "https://files.pythonhosted.org/packages/ce/6a/8b74c52ffd33dbbde00fe7251fee6a0acdc8cea33f7a43805aed258fb79b/simplejson-4.1.1-py3-none-any.whl", hash = "sha256:2ce92b3748f02423e26d2bfb636fb9d7a8f67c8f5854dcae69d350d123b2eee2", size = 69195, upload-time = "2026-04-24T19:24:57.962Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -1152,6 +1406,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "smmap" +version = "5.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1f/ea/49c993d6dfdd7338c9b1000a0f36817ed7ec84577ae2e52f890d1a4ff909/smmap-5.0.3.tar.gz", hash = "sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c", size = 22506, upload-time = "2026-03-09T03:43:26.1Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/d4/59e74daffcb57a07668852eeeb6035af9f32cbfd7a1d2511f17d2fe6a738/smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f", size = 24390, upload-time = "2026-03-09T03:43:24.361Z" }, +] + [[package]] name = "snowplow-tracker" version = "1.1.0" @@ -1165,6 +1428,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/10/1c76269cbf2d6e127f4415044d9ddb0295858230678bbf4bfba905593c82/snowplow_tracker-1.1.0-py3-none-any.whl", hash = "sha256:24ea32ddac9cca547421bf9ab162f5f33c00711c6ef118ad5f78093cee962224", size = 44128, upload-time = "2025-02-21T10:58:45.818Z" }, ] +[[package]] +name = "sqlglot" +version = "30.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/de/1542b04dcac0f85706fb8ce1ce7d2abcc230cdf10ea9e3aa7345393e5a90/sqlglot-30.11.0.tar.gz", hash = "sha256:1a23c6e2adb41da61fda46b1848d2fa26341d447fc0f0cd5ca21160362100991", size = 5893125, upload-time = "2026-06-11T17:11:37.845Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/86/53edf106e8cd3c883ccd0c6b470bf00ddf877a86e667665343b2d597329d/sqlglot-30.11.0-py3-none-any.whl", hash = "sha256:cffdee57d1f2f5472dc9f13087e618cf795841172b7d5ef78b63a051a52d2710", size = 698721, upload-time = "2026-06-11T17:11:35.737Z" }, +] + [[package]] name = "sqlparse" version = "0.5.5" @@ -1174,6 +1446,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/4b/359f28a903c13438ef59ebeee215fb25da53066db67b305c125f1c6d2a25/sqlparse-0.5.5-py3-none-any.whl", hash = "sha256:12a08b3bf3eec877c519589833aed092e2444e68240a3577e8e26148acc7b1ba", size = 46138, upload-time = "2025-12-19T07:17:46.573Z" }, ] +[[package]] +name = "tenacity" +version = "9.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, +] + [[package]] name = "text-unidecode" version = "1.3" @@ -1183,6 +1464,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", size = 78154, upload-time = "2019-08-30T21:37:03.543Z" }, ] +[[package]] +name = "tomlkit" +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/51/db/03eaf4331631ef6b27d6e3c9b68c54dc6f0d63d87201fed600cc409307fd/tomlkit-0.15.0.tar.gz", hash = "sha256:7d1a9ecba3086638211b13814ea79c90dd54dd11993564376f3aa92271f5c7a3", size = 161875, upload-time = "2026-05-10T07:38:22.245Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/43/8bd850ee71a191bf072e31302c73a66be413fecdd98fdcd111ecbcce13ca/tomlkit-0.15.0-py3-none-any.whl", hash = "sha256:4dbc8f0fc024412b57ced8757ac7461305126a648ff8c2c807fcb8e133a78738", size = 41328, upload-time = "2026-05-10T07:38:23.517Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0"