diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 49212916a3..98685f3ced 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -46,11 +46,11 @@ jobs: persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2 + uses: github/codeql-action/init@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3 with: languages: actions - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2 + uses: github/codeql-action/analyze@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3 with: category: "/language:actions" diff --git a/Makefile b/Makefile index d262de45a9..4fe761192c 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ # under the License. .PHONY: help install install-uv check-license lint \ test test-integration test-integration-setup test-integration-exec test-integration-cleanup test-integration-rebuild \ - test-s3 test-adls test-gcs test-coverage coverage-report \ + test-s3 test-adls test-gcs test-coverage coverage-report test test-notebook\ docs-serve docs-build notebook notebook-infra \ clean @@ -150,6 +150,9 @@ coverage-report: ## Combine and report coverage uv run $(PYTHON_ARG) coverage html uv run $(PYTHON_ARG) coverage xml +test-notebook: ## Run notebook tests (pyiceberg_example and spark_integration_example) via papermill + $(TEST_RUNNER) pytest tests/notebooks/test_pyiceberg_example.py tests/notebooks/test_spark_integration_example.py -m notebook $(PYTEST_ARGS) + # ================ # Documentation # ================ diff --git a/dev/docker-compose-integration.yml b/dev/docker-compose-integration.yml index 387e54ac2c..27c1b9a5f7 100644 --- a/dev/docker-compose-integration.yml +++ b/dev/docker-compose-integration.yml @@ -18,7 +18,10 @@ services: spark-iceberg: image: pyiceberg-spark:latest - build: spark/ + build: + context: spark/ + args: + ICEBERG_MAVEN_MIRROR: https://repository.apache.org/content/repositories/orgapacheiceberg-1282 container_name: pyiceberg-spark networks: iceberg_net: diff --git a/dev/spark/Dockerfile b/dev/spark/Dockerfile index 0e1f29d152..4a052f7dfa 100644 --- a/dev/spark/Dockerfile +++ b/dev/spark/Dockerfile @@ -19,11 +19,12 @@ FROM apache/spark:${BASE_IMAGE_SPARK_VERSION} # Dependency versions - keep these compatible # Changing these will invalidate the JAR download cache layer -ARG ICEBERG_VERSION=1.10.1 +ARG ICEBERG_VERSION=1.11.0 ARG ICEBERG_SPARK_RUNTIME_VERSION=4.0_2.13 ARG HADOOP_VERSION=3.4.1 ARG AWS_SDK_VERSION=2.24.6 ARG MAVEN_MIRROR=https://repo.maven.apache.org/maven2 +ARG ICEBERG_MAVEN_MIRROR=${MAVEN_MIRROR} USER root WORKDIR ${SPARK_HOME} @@ -38,11 +39,20 @@ RUN apt-get update -qq && \ # Download JARs with retry logic (most cacheable - only changes when versions change) # This is the slowest step, so we do it before copying config files +# Iceberg JARs use ICEBERG_MAVEN_MIRROR (defaults to MAVEN_MIRROR, can be overridden for staging repos) RUN set -e && \ cd "${SPARK_HOME}/jars" && \ for jar_path in \ "org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar" \ - "org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar" \ + "org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar"; \ + do \ + jar_name=$(basename "${jar_path}") && \ + curl -fsSL --retry 3 --retry-delay 5 \ + -o "${jar_name}" \ + "${ICEBERG_MAVEN_MIRROR}/${jar_path}" && \ + chown spark:spark "${jar_name}"; \ + done && \ + for jar_path in \ "org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar" \ "software/amazon/awssdk/bundle/${AWS_SDK_VERSION}/bundle-${AWS_SDK_VERSION}.jar"; \ do \ diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index 22d6b2e3c7..29d09e2604 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -1529,6 +1529,17 @@ catalog = load_catalog("default") catalog.view_exists("default.bar") ``` +## Register a view + +To register a view using existing metadata: + +```python +catalog.register_view( + identifier="docs_example.bids", + metadata_location="s3://warehouse/path/to/metadata.json" +) +``` + ## Table Statistics Management Manage table statistics with operations through the `Table` API: diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index 5db35ac3bd..ef3e51aea8 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -690,6 +690,22 @@ def update_namespace_properties( ValueError: If removals and updates have overlapping keys. """ + @abstractmethod + def register_view(self, identifier: str | Identifier, metadata_location: str) -> View: + """Register a new view using existing metadata. + + Args: + identifier (Union[str, Identifier]): View identifier for the view + metadata_location (str): The location to the metadata + + Returns: + View: The newly registered view + + Raises: + ViewAlreadyExistsError: If the view already exists. + TableAlreadyExistsError: If a table with the same name already exists. + """ + @abstractmethod def drop_view(self, identifier: str | Identifier) -> None: """Drop a view. diff --git a/pyiceberg/catalog/bigquery_metastore.py b/pyiceberg/catalog/bigquery_metastore.py index 1b8a172e25..d389e2e1df 100644 --- a/pyiceberg/catalog/bigquery_metastore.py +++ b/pyiceberg/catalog/bigquery_metastore.py @@ -305,6 +305,9 @@ def register_table(self, identifier: str | Identifier, metadata_location: str, o def list_views(self, namespace: str | Identifier) -> list[Identifier]: raise NotImplementedError + def register_view(self, identifier: str | Identifier, metadata_location: str) -> View: + raise NotImplementedError + def drop_view(self, identifier: str | Identifier) -> None: raise NotImplementedError diff --git a/pyiceberg/catalog/dynamodb.py b/pyiceberg/catalog/dynamodb.py index aa66f08bcf..2aaf11c8a5 100644 --- a/pyiceberg/catalog/dynamodb.py +++ b/pyiceberg/catalog/dynamodb.py @@ -553,6 +553,9 @@ def create_view( def list_views(self, namespace: str | Identifier) -> list[Identifier]: raise NotImplementedError + def register_view(self, identifier: str | Identifier, metadata_location: str) -> View: + raise NotImplementedError + def drop_view(self, identifier: str | Identifier) -> None: raise NotImplementedError diff --git a/pyiceberg/catalog/glue.py b/pyiceberg/catalog/glue.py index a21fe6da3c..92d0aa4554 100644 --- a/pyiceberg/catalog/glue.py +++ b/pyiceberg/catalog/glue.py @@ -970,6 +970,9 @@ def create_view( def list_views(self, namespace: str | Identifier) -> list[Identifier]: raise NotImplementedError + def register_view(self, identifier: str | Identifier, metadata_location: str) -> View: + raise NotImplementedError + def drop_view(self, identifier: str | Identifier) -> None: raise NotImplementedError diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index afca5954b1..8d2a743084 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -854,6 +854,9 @@ def update_namespace_properties( return PropertiesUpdateSummary(removed=list(removed or []), updated=list(updated or []), missing=list(expected_to_change)) + def register_view(self, identifier: str | Identifier, metadata_location: str) -> View: + raise NotImplementedError + def drop_view(self, identifier: str | Identifier) -> None: raise NotImplementedError diff --git a/pyiceberg/catalog/noop.py b/pyiceberg/catalog/noop.py index 1524343610..3a1a8e7c47 100644 --- a/pyiceberg/catalog/noop.py +++ b/pyiceberg/catalog/noop.py @@ -132,6 +132,9 @@ def view_exists(self, identifier: str | Identifier) -> bool: def namespace_exists(self, namespace: str | Identifier) -> bool: raise NotImplementedError + def register_view(self, identifier: str | Identifier, metadata_location: str) -> View: + raise NotImplementedError + def drop_view(self, identifier: str | Identifier) -> None: raise NotImplementedError diff --git a/pyiceberg/catalog/rest/__init__.py b/pyiceberg/catalog/rest/__init__.py index b3a80e11aa..8c5647e7de 100644 --- a/pyiceberg/catalog/rest/__init__.py +++ b/pyiceberg/catalog/rest/__init__.py @@ -154,6 +154,7 @@ class Endpoints: list_views: str = "namespaces/{namespace}/views" load_view: str = "namespaces/{namespace}/views/{view}" create_view: str = "namespaces/{namespace}/views" + register_view: str = "namespaces/{namespace}/register-view" drop_view: str = "namespaces/{namespace}/views/{view}" view_exists: str = "namespaces/{namespace}/views/{view}" plan_table_scan: str = "namespaces/{namespace}/tables/{table}/plan" @@ -183,6 +184,7 @@ class Capability: V1_LIST_VIEWS = Endpoint(http_method=HttpMethod.GET, path=f"{API_PREFIX}/{Endpoints.list_views}") V1_LOAD_VIEW = Endpoint(http_method=HttpMethod.GET, path=f"{API_PREFIX}/{Endpoints.load_view}") V1_VIEW_EXISTS = Endpoint(http_method=HttpMethod.HEAD, path=f"{API_PREFIX}/{Endpoints.view_exists}") + V1_REGISTER_VIEW = Endpoint(http_method=HttpMethod.POST, path=f"{API_PREFIX}/{Endpoints.register_view}") V1_DELETE_VIEW = Endpoint(http_method=HttpMethod.DELETE, path=f"{API_PREFIX}/{Endpoints.drop_view}") V1_SUBMIT_TABLE_SCAN_PLAN = Endpoint(http_method=HttpMethod.POST, path=f"{API_PREFIX}/{Endpoints.plan_table_scan}") V1_TABLE_SCAN_PLAN_TASKS = Endpoint(http_method=HttpMethod.POST, path=f"{API_PREFIX}/{Endpoints.fetch_scan_tasks}") @@ -322,6 +324,11 @@ class RegisterTableRequest(IcebergBaseModel): overwrite: bool +class RegisterViewRequest(IcebergBaseModel): + name: str + metadata_location: str = Field(..., alias="metadata-location") + + class ConfigResponse(IcebergBaseModel): defaults: Properties | None = Field(default_factory=dict) overrides: Properties | None = Field(default_factory=dict) @@ -1332,6 +1339,29 @@ def view_exists(self, identifier: str | Identifier) -> bool: return False + @retry(**_RETRY_ARGS) + def register_view(self, identifier: str | Identifier, metadata_location: str) -> View: + self._check_endpoint(Capability.V1_REGISTER_VIEW) + namespace_and_view = self._split_identifier_for_path(identifier, IdentifierKind.VIEW) + namespace = namespace_and_view["namespace"] + view = namespace_and_view["view"] + if self.table_exists(identifier): + raise TableAlreadyExistsError(f"Table {namespace}.{view} already exists") + + request = RegisterViewRequest(name=view, metadata_location=metadata_location) + serialized_json = request.model_dump_json().encode(UTF8) + response = self._session.post( + self.url(Endpoints.register_view, namespace=namespace), + data=serialized_json, + ) + try: + response.raise_for_status() + except HTTPError as exc: + _handle_non_200_response(exc, {409: ViewAlreadyExistsError}) + + view_response = ViewResponse.model_validate_json(response.text) + return self._response_to_view(self.identifier_to_tuple(identifier), view_response) + @retry(**_RETRY_ARGS) def drop_view(self, identifier: str) -> None: self._check_endpoint(Capability.V1_DELETE_VIEW) diff --git a/pyiceberg/catalog/sql.py b/pyiceberg/catalog/sql.py index dc703e95a4..92ac53751f 100644 --- a/pyiceberg/catalog/sql.py +++ b/pyiceberg/catalog/sql.py @@ -745,6 +745,9 @@ def list_views(self, namespace: str | Identifier) -> list[Identifier]: def view_exists(self, identifier: str | Identifier) -> bool: raise NotImplementedError + def register_view(self, identifier: str | Identifier, metadata_location: str) -> View: + raise NotImplementedError + def drop_view(self, identifier: str | Identifier) -> None: raise NotImplementedError diff --git a/pyiceberg/table/inspect.py b/pyiceberg/table/inspect.py index 5da343ccb6..ac8e627d7b 100644 --- a/pyiceberg/table/inspect.py +++ b/pyiceberg/table/inspect.py @@ -404,6 +404,7 @@ def _get_all_manifests_schema(self) -> pa.Schema: all_manifests_schema = self._get_manifests_schema() all_manifests_schema = all_manifests_schema.append(pa.field("reference_snapshot_id", pa.int64(), nullable=False)) + all_manifests_schema = all_manifests_schema.append(pa.field("key_metadata", pa.binary(), nullable=True)) return all_manifests_schema def _generate_manifests_table(self, snapshot: Snapshot | None, is_all_manifests_table: bool = False) -> pa.Table: @@ -468,6 +469,7 @@ def _partition_summaries_to_rows( } if is_all_manifests_table: manifest_row["reference_snapshot_id"] = snapshot.snapshot_id + manifest_row["key_metadata"] = manifest.key_metadata manifests.append(manifest_row) return pa.Table.from_pylist( diff --git a/pyproject.toml b/pyproject.toml index ac1177db44..96118f8451 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,6 +122,9 @@ dev = [ "google-cloud-bigquery>=3.33.0,<4", "pyarrow-stubs>=20.0.0.20251107", # Remove when pyarrow >= 23.0.0 https://github.com/apache/arrow/pull/47609 "sqlalchemy>=2.0.18,<3", + "papermill>=2.6.0", + "nbformat>=5.10.0", + "ipykernel>=6.29.0", ] # for mkdocs docs = [ @@ -161,6 +164,7 @@ markers = [ "integration: marks integration tests against Apache Spark", "gcs: marks a test as requiring access to gcs compliant storage (use with --gs.token, --gs.project, and --gs.endpoint)", "benchmark: collection of tests to validate read/write performance before and after a change", + "notebook: marks tests that execute Jupyter notebooks via papermill", ] # Turns a warning into an error diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index bd77f8ad78..edbe67d974 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -105,6 +105,7 @@ Capability.V1_LIST_VIEWS, Capability.V1_LOAD_VIEW, Capability.V1_VIEW_EXISTS, + Capability.V1_REGISTER_VIEW, Capability.V1_DELETE_VIEW, Capability.V1_SUBMIT_TABLE_SCAN_PLAN, Capability.V1_TABLE_SCAN_PLAN_TASKS, @@ -2182,6 +2183,72 @@ def test_table_identifier_in_commit_table_request( ) +def test_register_view_200(rest_mock: Mocker, example_view_metadata_rest_json: dict[str, Any]) -> None: + rest_mock.head( + f"{TEST_URI}v1/namespaces/default/tables/registered_view", + status_code=404, + request_headers=TEST_HEADERS, + ) + rest_mock.post( + f"{TEST_URI}v1/namespaces/default/register-view", + json=example_view_metadata_rest_json, + status_code=200, + request_headers=TEST_HEADERS, + ) + + catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN) + actual = catalog.register_view( + identifier=("default", "registered_view"), metadata_location="s3://warehouse/database/view/metadata.json" + ) + expected = View( + identifier=("default", "registered_view"), + metadata=ViewMetadata(**example_view_metadata_rest_json["metadata"]), + ) + assert actual == expected + + +def test_register_view_409_view(rest_mock: Mocker) -> None: + rest_mock.head( + f"{TEST_URI}v1/namespaces/default/tables/registered_view", + status_code=404, + request_headers=TEST_HEADERS, + ) + rest_mock.post( + f"{TEST_URI}v1/namespaces/default/register-view", + json={ + "error": { + "message": "View already exists: default.view in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e", + "type": "AlreadyExistsException", + "code": 409, + } + }, + status_code=409, + request_headers=TEST_HEADERS, + ) + + catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN) + with pytest.raises(ViewAlreadyExistsError) as e: + catalog.register_view( + identifier=("default", "registered_view"), metadata_location="s3://warehouse/database/view/metadata.json" + ) + assert "View already exists" in str(e.value) + + +def test_register_view_409_table(rest_mock: Mocker) -> None: + rest_mock.head( + f"{TEST_URI}v1/namespaces/default/tables/registered_view", + status_code=200, + request_headers=TEST_HEADERS, + ) + + catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN) + with pytest.raises(TableAlreadyExistsError) as e: + catalog.register_view( + identifier=("default", "registered_view"), metadata_location="s3://warehouse/database/view/metadata.json" + ) + assert "Table default.registered_view already exists" in str(e.value) + + def test_drop_view_invalid_namespace(rest_mock: Mocker) -> None: view = "view" with pytest.raises(NoSuchIdentifierError) as e: diff --git a/tests/integration/test_inspect_table.py b/tests/integration/test_inspect_table.py index 03d4437d18..4d8dfbe9bb 100644 --- a/tests/integration/test_inspect_table.py +++ b/tests/integration/test_inspect_table.py @@ -1012,6 +1012,7 @@ def test_inspect_all_manifests(spark: SparkSession, session_catalog: Catalog, fo "deleted_delete_files_count", "partition_summaries", "reference_snapshot_id", + "key_metadata", ] int_cols = [ diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index 0a09867656..b6fc7067f6 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -2027,7 +2027,7 @@ def test_write_optional_list(session_catalog: Catalog) -> None: required=False, ), ) - session_catalog.create_table_if_not_exists(identifier, schema) + _create_table(session_catalog, identifier, schema=schema) df_1 = pa.Table.from_pylist( [ diff --git a/tests/notebooks/test_pyiceberg_example.py b/tests/notebooks/test_pyiceberg_example.py new file mode 100644 index 0000000000..eea5b49963 --- /dev/null +++ b/tests/notebooks/test_pyiceberg_example.py @@ -0,0 +1,101 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pathlib import Path + +import nbformat +import papermill as pm +import pytest + +pytestmark = pytest.mark.notebook + +NOTEBOOK_PATH = Path(__file__).parents[2] / "notebooks" / "pyiceberg_example.ipynb" + + +def get_all_stdout(nb: nbformat.NotebookNode) -> str: + """Concatenate all stdout streams from every executed cell.""" + return "".join( + out.get("text", "") + for cell in nb.cells + for out in cell.get("outputs", []) + if out.get("output_type") == "stream" and out.get("name") == "stdout" + ) + + +@pytest.fixture(scope="session") +def pyiceberg_nb(tmp_path_factory: pytest.TempPathFactory) -> nbformat.NotebookNode: + out = tmp_path_factory.mktemp("nb_out") / "pyiceberg_example_out.ipynb" + return pm.execute_notebook(str(NOTEBOOK_PATH), str(out), kernel_name="python3") + + +class TestSmoke: + def test_notebook_completes_without_error(self, pyiceberg_nb: nbformat.NotebookNode) -> None: + """papermill raises PapermillExecutionError if any cell fails.""" + assert pyiceberg_nb is not None + + def test_all_code_cells_executed(self, pyiceberg_nb: nbformat.NotebookNode) -> None: + for cell in pyiceberg_nb.cells: + if cell.cell_type == "code": + assert cell.get("execution_count") is not None, f"Cell not executed:\n{cell.source[:80]}" + + +class TestCellOutputs: + def test_pyiceberg_version_printed(self, pyiceberg_nb: nbformat.NotebookNode) -> None: + assert "PyIceberg version:" in get_all_stdout(pyiceberg_nb) + + def test_warehouse_location_printed(self, pyiceberg_nb: nbformat.NotebookNode) -> None: + stdout = get_all_stdout(pyiceberg_nb) + assert "Warehouse location:" in stdout + assert "iceberg_warehouse_" in stdout + + def test_catalog_loaded_successfully(self, pyiceberg_nb: nbformat.NotebookNode) -> None: + assert "Catalog loaded successfully!" in get_all_stdout(pyiceberg_nb) + + def test_namespace_default_created(self, pyiceberg_nb: nbformat.NotebookNode) -> None: + assert "default" in get_all_stdout(pyiceberg_nb) + + def test_rows_written_is_five(self, pyiceberg_nb: nbformat.NotebookNode) -> None: + assert "Rows written: 5" in get_all_stdout(pyiceberg_nb) + + def test_schema_evolved_message(self, pyiceberg_nb: nbformat.NotebookNode) -> None: + assert "Schema evolved!" in get_all_stdout(pyiceberg_nb) + + def test_tip_per_mile_column_present_after_evolution(self, pyiceberg_nb: nbformat.NotebookNode) -> None: + assert "tip_per_mile" in get_all_stdout(pyiceberg_nb) + + def test_filter_result_is_positive(self, pyiceberg_nb: nbformat.NotebookNode) -> None: + """The notebook prints 'Rows with tip_per_mile > 1.0: N' — N must be > 0.""" + stdout = get_all_stdout(pyiceberg_nb) + assert "Rows with tip_per_mile > 1.0:" in stdout + for line in stdout.splitlines(): + if "Rows with tip_per_mile > 1.0:" in line: + count = int(line.split(":")[-1].strip()) + assert count > 0 + break + + def test_snapshot_id_printed(self, pyiceberg_nb: nbformat.NotebookNode) -> None: + assert "Current snapshot ID:" in get_all_stdout(pyiceberg_nb) + + def test_table_history_has_entries(self, pyiceberg_nb: nbformat.NotebookNode) -> None: + stdout = get_all_stdout(pyiceberg_nb) + assert "Table history:" in stdout + assert "Snapshot:" in stdout + + def test_warehouse_contains_parquet_and_metadata_files(self, pyiceberg_nb: nbformat.NotebookNode) -> None: + stdout = get_all_stdout(pyiceberg_nb) + assert ".parquet" in stdout + assert ".metadata.json" in stdout diff --git a/tests/notebooks/test_spark_integration_example.py b/tests/notebooks/test_spark_integration_example.py new file mode 100644 index 0000000000..e242e43157 --- /dev/null +++ b/tests/notebooks/test_spark_integration_example.py @@ -0,0 +1,170 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import textwrap +from pathlib import Path + +import nbformat +import papermill as pm +import pytest + +pytestmark = pytest.mark.notebook + +NOTEBOOK_PATH = Path(__file__).parents[2] / "notebooks" / "spark_integration_example.ipynb" + +# --------------------------------------------------------------------------- +# Mock pyspark +# Replaces pyspark.sql.SparkSession with a fake one +# --------------------------------------------------------------------------- +_MOCK_PYSPARK = textwrap.dedent("""\ + import sys + import types + from unittest.mock import MagicMock + + def _make_fake_pyspark(): + pyspark_mod = types.ModuleType("pyspark") + sql_mod = types.ModuleType("pyspark.sql") + pyspark_mod.sql = sql_mod + sys.modules.setdefault("pyspark", pyspark_mod) + sys.modules.setdefault("pyspark.sql", sql_mod) + return pyspark_mod, sql_mod + + _pyspark, _sql = _make_fake_pyspark() + + _SHOW_CATALOGS = ( + "+-------------+\\n" + "|catalogName |\\n" + "+-------------+\\n" + "|spark_catalog|\\n" + "|local |\\n" + "+-------------+\\n" + ) + _SHOW_NAMESPACES = ( + "+---------+\\n" + "|namespace|\\n" + "+---------+\\n" + "|default |\\n" + "+---------+\\n" + ) + _SHOW_TABLES = ( + "+---------+-----------+-----------+\\n" + "|namespace|tableName |isTemporary|\\n" + "+---------+-----------+-----------+\\n" + "|default |test_all |false |\\n" + "+---------+-----------+-----------+\\n" + ) + _DESCRIBE_TABLE = ( + "+--------------------+---------+-------+\\n" + "|col_name |data_type|comment|\\n" + "+--------------------+---------+-------+\\n" + "|boolean_col |boolean |null |\\n" + "|integer_col |integer |null |\\n" + "+--------------------+---------+-------+\\n" + ) + _SQL_RESPONSES = { + "SHOW CATALOGS": _SHOW_CATALOGS, + "SHOW NAMESPACES": _SHOW_NAMESPACES, + "SHOW TABLES FROM default": _SHOW_TABLES, + "DESCRIBE TABLE default.test_all_types": _DESCRIBE_TABLE, + } + + def _make_df(output): + df = MagicMock() + df.show.side_effect = lambda *a, **kw: print(output, end="") + return df + + class _FakeBuilder: + def remote(self, url): return self + def getOrCreate(self): return _FakeSession() + + class _FakeSession: + builder = _FakeBuilder() + def sql(self, query): + key = query.strip().rstrip(";") + output = _SQL_RESPONSES.get(key, "+------+\\n| col |\\n+------+\\n| val |\\n+------+\\n") + return _make_df(output) + + _FakeSparkSession = MagicMock(spec=object) + _FakeSparkSession.builder = _FakeBuilder() + _sql.SparkSession = _FakeSparkSession +""") + + +def get_all_stdout(nb: nbformat.NotebookNode) -> str: + """Concatenate all stdout streams from every executed cell.""" + return "".join( + out.get("text", "") + for cell in nb.cells + for out in cell.get("outputs", []) + if out.get("output_type") == "stream" and out.get("name") == "stdout" + ) + + +def _inject_mock_and_execute(notebook_path: Path, output_path: Path) -> nbformat.NotebookNode: + """ + Load the real notebook, prepend the mock-pyspark setup cell, write to a + temporary copy and execute it with papermill. + """ + nb = nbformat.read(str(notebook_path), as_version=4) + + mock_cell = nbformat.v4.new_code_cell(_MOCK_PYSPARK) + mock_cell.metadata["tags"] = ["injected-mock"] + nb.cells.insert(0, mock_cell) + + patched_path = output_path.parent / "spark_patched.ipynb" + nbformat.write(nb, str(patched_path)) + + return pm.execute_notebook(str(patched_path), str(output_path), kernel_name="python3") + + +@pytest.fixture(scope="session") +def spark_nb(tmp_path_factory: pytest.TempPathFactory) -> nbformat.NotebookNode: + out = tmp_path_factory.mktemp("nb_out") / "spark_integration_example_out.ipynb" + return _inject_mock_and_execute(NOTEBOOK_PATH, out) + + +class TestSmoke: + def test_notebook_completes_without_error(self, spark_nb: nbformat.NotebookNode) -> None: + assert spark_nb is not None + + def test_all_code_cells_executed(self, spark_nb: nbformat.NotebookNode) -> None: + for cell in spark_nb.cells: + if cell.cell_type == "code": + assert cell.get("execution_count") is not None, f"Cell not executed:\n{cell.source[:80]}" + + +class TestCellOutputs: + def test_show_catalogs_lists_spark_catalog_and_local(self, spark_nb: nbformat.NotebookNode) -> None: + stdout = get_all_stdout(spark_nb) + assert "spark_catalog" in stdout + assert "local" in stdout + + def test_show_namespaces_contains_default(self, spark_nb: nbformat.NotebookNode) -> None: + assert "default" in get_all_stdout(spark_nb) + + def test_show_tables_produces_tabular_output(self, spark_nb: nbformat.NotebookNode) -> None: + assert "+---------+-----------+-----------+" in get_all_stdout(spark_nb) + + def test_describe_table_lists_column_names(self, spark_nb: nbformat.NotebookNode) -> None: + assert "col_name" in get_all_stdout(spark_nb) + + def test_describe_table_lists_data_types(self, spark_nb: nbformat.NotebookNode) -> None: + stdout = get_all_stdout(spark_nb) + assert "boolean" in stdout or "integer" in stdout + + def test_show_tables_includes_test_table_row(self, spark_nb: nbformat.NotebookNode) -> None: + assert "test_all" in get_all_stdout(spark_nb) diff --git a/uv.lock b/uv.lock index aa41d579e3..2f67510b19 100644 --- a/uv.lock +++ b/uv.lock @@ -614,11 +614,11 @@ virtualenv = [ [[package]] name = "cachetools" -version = "7.0.6" +version = "7.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/76/7b/1755ed2c6bfabd1d98b37ae73152f8dcf94aa40fee119d163c19ed484704/cachetools-7.0.6.tar.gz", hash = "sha256:e5d524d36d65703a87243a26ff08ad84f73352adbeafb1cde81e207b456aaf24", size = 37526, upload-time = "2026-04-20T19:02:23.289Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/1d/7cbe8f84352789be3dcbb1239f425984f353efa253deeeff38af600870b3/cachetools-7.1.0.tar.gz", hash = "sha256:ea5406e92956f9006b121f8032177c6b02cc5f9a488d1e53b2e4d9cb5aae15c6", size = 40085, upload-time = "2026-05-01T21:20:04.649Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/c4/cf76242a5da1410917107ff14551764aa405a5fd10cd10cf9a5ca8fa77f4/cachetools-7.0.6-py3-none-any.whl", hash = "sha256:4e94956cfdd3086f12042cdd29318f5ced3893014f7d0d059bf3ead3f85b7f8b", size = 13976, upload-time = "2026-04-20T19:02:21.187Z" }, + { url = "https://files.pythonhosted.org/packages/87/42/e09974bc74ea2791f985dff3d7f06065d11abcc31da6c46fc5f83046516e/cachetools-7.1.0-py3-none-any.whl", hash = "sha256:05afd1d309309e7c8971db462b4cf516d93fa8c9aea1b906e26e61b4399d0d82", size = 16748, upload-time = "2026-05-01T21:20:02.879Z" }, ] [[package]] @@ -1372,6 +1372,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7a/e3/9d34173ec068631faea3ea6e73050700729363e7e33306a9a3218e5cdc61/duckdb-1.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:c9f3e0b71b8a50fccfb42794899285d9d318ce2503782b9dd54868e5ecd0ad31", size = 14402513, upload-time = "2026-04-13T11:30:06.609Z" }, ] +[[package]] +name = "entrypoints" +version = "0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ea/8d/a7121ffe5f402dc015277d2d31eb82d2187334503a011c18f2e78ecbb9b2/entrypoints-0.4.tar.gz", hash = "sha256:b706eddaa9218a19ebcd67b56818f05bb27589b1ca9e8d797b74affad4ccacd4", size = 13974, upload-time = "2022-02-02T21:30:28.172Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/a8/365059bbcd4572cbc41de17fd5b682be5868b218c3c5479071865cab9078/entrypoints-0.4-py3-none-any.whl", hash = "sha256:f174b5ff827504fd3cd97cc3f8649f3693f51538c7e4bdf3ef002c8429d42f9f", size = 5294, upload-time = "2022-02-02T21:30:26.024Z" }, +] + [[package]] name = "exceptiongroup" version = "1.3.0" @@ -1771,15 +1780,15 @@ grpc = [ [[package]] name = "google-auth" -version = "2.49.2" +version = "2.50.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography" }, { name = "pyasn1-modules" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c6/fc/e925290a1ad95c975c459e2df070fac2b90954e13a0370ac505dff78cb99/google_auth-2.49.2.tar.gz", hash = "sha256:c1ae38500e73065dcae57355adb6278cf8b5c8e391994ae9cbadbcb9631ab409", size = 333958, upload-time = "2026-04-10T00:41:21.888Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/18/238d7021d151bdab868f23433817b027dd759135202f4dfce0670d1230ca/google_auth-2.50.0.tar.gz", hash = "sha256:f35eafb191195328e8ce10a7883970877e7aeb49c2bfaa54aa0e394316d353d0", size = 336523, upload-time = "2026-04-30T21:19:29.659Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/73/76/d241a5c927433420507215df6cac1b1fa4ac0ba7a794df42a84326c68da8/google_auth-2.49.2-py3-none-any.whl", hash = "sha256:c2720924dfc82dedb962c9f52cabb2ab16714fd0a6a707e40561d217574ed6d5", size = 240638, upload-time = "2026-04-10T00:41:14.501Z" }, + { url = "https://files.pythonhosted.org/packages/37/cf/4880c2137c14280b2f59975cdf12cc442bc0ae1f9ea473a26eaa0c146786/google_auth-2.50.0-py3-none-any.whl", hash = "sha256:04382175e28b94f49694977f0a792688b59a668def1499e9d8de996dc9ce5b15", size = 246495, upload-time = "2026-04-30T21:19:27.664Z" }, ] [[package]] @@ -2184,7 +2193,7 @@ wheels = [ [[package]] name = "huggingface-hub" -version = "1.12.0" +version = "1.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock" }, @@ -2197,9 +2206,9 @@ dependencies = [ { name = "typer" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/56/52/1b54cb569509c725a32c1315261ac9fd0e6b91bbbf74d86fca10d3376164/huggingface_hub-1.12.0.tar.gz", hash = "sha256:7c3fe85e24b652334e5d456d7a812cd9a071e75630fac4365d9165ab5e4a34b6", size = 763091, upload-time = "2026-04-24T13:32:08.674Z" } +sdist = { url = "https://files.pythonhosted.org/packages/89/ff/ec7ed2eb43bd7ce8bb2233d109cc235c3e807ffe5e469dc09db261fac05e/huggingface_hub-1.13.0.tar.gz", hash = "sha256:f6df2dac5abe82ce2fe05873d10d5ff47bc677d616a2f521f4ee26db9415d9d0", size = 781788, upload-time = "2026-04-30T11:57:33.858Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/2b/ef03ddb96bd1123503c2bd6932001020292deea649e9bf4caa2cb65a85bf/huggingface_hub-1.12.0-py3-none-any.whl", hash = "sha256:d74939969585ee35748bd66de09baf84099d461bda7287cd9043bfb99b0e424d", size = 646806, upload-time = "2026-04-24T13:32:06.717Z" }, + { url = "https://files.pythonhosted.org/packages/93/db/4b1cdae9460ae1f3ca020cd767f013430ce23eb1d9c890ae3a0609b38d26/huggingface_hub-1.13.0-py3-none-any.whl", hash = "sha256:e942cb50d6a08dd5306688b1ac05bda157fd2fcc88b63dae405f7bd0d3234005", size = 660643, upload-time = "2026-04-30T11:57:31.802Z" }, ] [[package]] @@ -3524,26 +3533,26 @@ wheels = [ [[package]] name = "mypy-boto3-dynamodb" -version = "1.42.73" +version = "1.43.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/70/a2/fc93b44ed07322ede4cc45c6c66370ff53ad8a195b95f9f80d1edbb7735a/mypy_boto3_dynamodb-1.42.73.tar.gz", hash = "sha256:3fd9e093a8c982a53d55f6ac3d109d89957fe82b5c7ece65f27b772ed7a5ca9b", size = 48723, upload-time = "2026-03-20T19:59:32.033Z" } +sdist = { url = "https://files.pythonhosted.org/packages/38/41/2823aad9f3abab6104cd8aaafebabfba19927227fbc277e2289c3b915c1d/mypy_boto3_dynamodb-1.43.0.tar.gz", hash = "sha256:f0cea38e058f1d07361ecb55d8f40665d824b42cf4864724c7fccc8bf3946fcd", size = 48755, upload-time = "2026-04-29T22:59:59.119Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/41/f4ececbceb13d1c2fbc1cfee56d65231c98b19f4d0b41baf3ab5d88b6863/mypy_boto3_dynamodb-1.42.73-py3-none-any.whl", hash = "sha256:2df84f71ff41fba5f12e803be19c4483daa64ba030fc9d62ba5e5f9f09a4def8", size = 58828, upload-time = "2026-03-20T19:59:24.878Z" }, + { url = "https://files.pythonhosted.org/packages/be/e2/1f63f04600fc21cb848c10ce16490252e27d812bfd5a716659283049d3a3/mypy_boto3_dynamodb-1.43.0-py3-none-any.whl", hash = "sha256:60b64d15e86d406a980d96f734d8c7fb1704668d0234dc8dabd2532c902a3ba6", size = 58873, upload-time = "2026-04-29T22:59:56.823Z" }, ] [[package]] name = "mypy-boto3-glue" -version = "1.42.89" +version = "1.43.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/72/90/38de4d3044a01b0cd847eba569b7ad6959fc982401b70128da812d1afb0d/mypy_boto3_glue-1.42.89.tar.gz", hash = "sha256:53e373634d8eca147f19b672fb661edfe160015cc5fd44cd8541f77f2f5298c7", size = 139751, upload-time = "2026-04-13T19:51:29.339Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d5/1c/f2a6db68812e0570a6ff7b4897914d20a52e737ed3b7d8bb484f602ef860/mypy_boto3_glue-1.43.0.tar.gz", hash = "sha256:dfbc20c54a2cc204591a94d8a655ed223c811303b9eda13395c4cec20f20fe98", size = 139736, upload-time = "2026-04-29T23:01:07.275Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c4/81/aca9b9162f6b762b23b2a940061cd4a22f7e3a3c458ed87f6f67321ac13f/mypy_boto3_glue-1.42.89-py3-none-any.whl", hash = "sha256:7cb9643dda708106031a8c63b6160e37094d5697ac36a51904b60934d1004023", size = 141324, upload-time = "2026-04-13T19:51:26.033Z" }, + { url = "https://files.pythonhosted.org/packages/cb/53/45bb964ea03ed76d3b9575ee02e4bc01af0ad5b5cb15d843921a4f5675b7/mypy_boto3_glue-1.43.0-py3-none-any.whl", hash = "sha256:1a64f69edaec37b69d64523a4c16ce03422fc90f4d8c6c231fefdfccc32c730f", size = 141332, upload-time = "2026-04-29T23:01:04.53Z" }, ] [[package]] @@ -3896,6 +3905,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ef/af/4fbc8cab944db5d21b7e2a5b8e9211a03a79852b1157e2c102fcc61ac440/pandocfilters-1.5.1-py2.py3-none-any.whl", hash = "sha256:93be382804a9cdb0a7267585f157e5d1731bbe5545a85b268d6f5fe6232de2bc", size = 8663, upload-time = "2024-01-18T20:08:11.28Z" }, ] +[[package]] +name = "papermill" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp", marker = "python_full_version == '3.12.*'" }, + { name = "click" }, + { name = "entrypoints" }, + { name = "nbclient" }, + { name = "nbformat" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "tenacity" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f8/b6/92d770c5ced66ed0134256f8de781e98c824d3a0662af1643a91fcc36663/papermill-2.7.0.tar.gz", hash = "sha256:ec10b37594a060662f57269e1ebd108c209d204450f00fdfeb70a1c7cfb7fbc8", size = 77961, upload-time = "2026-02-27T19:07:30.548Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/9f/f9fd57a727dcc89c54e84455d8317bff7db05ef21bb6d05b03705111f7c0/papermill-2.7.0-py3-none-any.whl", hash = "sha256:e1855e6670100a02bb4f8a6870484a5c10b84a8d2e49c49921c90209940c7514", size = 88858, upload-time = "2026-02-27T19:07:28.862Z" }, +] + [[package]] name = "parso" version = "0.8.5" @@ -3992,26 +4021,26 @@ wheels = [ [[package]] name = "prek" -version = "0.3.10" +version = "0.3.11" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/31/3e68cd8f45cb5f3f81009ff42d3a3cde0887b83f214a4eb0ded9ae239cc2/prek-0.3.10.tar.gz", hash = "sha256:f4e9c533612bbaa9f89eca0e80ab6e59a05b0fd15ca7c6642a35bb303731ad6f", size = 425926, upload-time = "2026-04-21T11:29:37.122Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/60/5b980c70525ca5f0d17942d8eae13b399051aa384413366fe5df229712ea/prek-0.3.11.tar.gz", hash = "sha256:c4cf77848009503c58d80ff216e32af45b63ea49652bb5546748c1ebfd4d9847", size = 433440, upload-time = "2026-04-27T04:22:59.923Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/62/b7/306d57aaaf503be1ee58929f9e58e812fbde9bea52c1a578e4996d82425a/prek-0.3.10-py3-none-linux_armv6l.whl", hash = "sha256:7250661f003d902b7b601141d64c7015f93eeeeeb1c485f714374582714d7c51", size = 5416913, upload-time = "2026-04-21T11:29:33.716Z" }, - { url = "https://files.pythonhosted.org/packages/93/ec/4b4b60d17c5eb639779a059d7f66a8fff5b38c7cb26c9c5b3a853cc6a9c8/prek-0.3.10-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9da699368bb11e85fc58b57ae4c1ccc703364b1d95b595bd17a27b27fa6df50d", size = 5784725, upload-time = "2026-04-21T11:29:38.356Z" }, - { url = "https://files.pythonhosted.org/packages/f3/c3/6d4dcc9446a1630a888e3f105c4775536afe58c20f18833fec8e0d841ac0/prek-0.3.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:f245fa9935cee6ea7d5ba1c6e7dd7546ccf6f01fb7d604bb6a344c8e3b49a9c2", size = 5360071, upload-time = "2026-04-21T11:29:10.743Z" }, - { url = "https://files.pythonhosted.org/packages/36/2d/d1f63ca15f4a275fcdc712fd9ef64787abb4ba86f2e79986a26108a5f1e7/prek-0.3.10-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:d321cad394ef4436ed9fce74150076a5eefabe310ed01b3f5735746284b9d046", size = 5615550, upload-time = "2026-04-21T11:29:30.985Z" }, - { url = "https://files.pythonhosted.org/packages/7c/1b/a925050ba30791e50e7690a6dabe33dc08a0133e7efab4b9d91088261d40/prek-0.3.10-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25e80017ecf06a11bf9c8ebcb67d38183e0abf2f18e012ee6a5f91d1b9c065d6", size = 5338159, upload-time = "2026-04-21T11:29:25.314Z" }, - { url = "https://files.pythonhosted.org/packages/56/ec/7d563a333198ead10f9a13fcf72ca910a59135b5d58429ed3b1fff9a0ff9/prek-0.3.10-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba24b4c8e40cb9ae96d58efa3eb91813cdff4f10b78b79201c93a0dc6b8763e2", size = 5728600, upload-time = "2026-04-21T11:29:23.687Z" }, - { url = "https://files.pythonhosted.org/packages/4c/88/10d493ef308f10321b92bb4e65ab60030b3c5c6304790c678be249e383e3/prek-0.3.10-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aac1c6f8dabb0a202a27516bf78cbfbe8a05b7cfebe2078ff6d0e12e7c77c7d5", size = 6610055, upload-time = "2026-04-21T11:29:27.398Z" }, - { url = "https://files.pythonhosted.org/packages/c4/14/552b239d99fdd09fd6664b69089359340778f84a9f866a1fecc38b66c811/prek-0.3.10-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a9a446e178562e2c3cfc71c8f0b21043b571209533a104042b6b9e36662849e", size = 6007394, upload-time = "2026-04-21T11:29:14.466Z" }, - { url = "https://files.pythonhosted.org/packages/21/80/12b9de6b6721a27f57ed633a8864435ec2fa7535cba7f48f3ea4a52ce683/prek-0.3.10-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:d7344b2acb88b52592c0e0f3dd33349f75e6a4813d3bb44385aa8c70084990ca", size = 5616625, upload-time = "2026-04-21T11:29:29.511Z" }, - { url = "https://files.pythonhosted.org/packages/69/02/7235f6b6d65a3e6e036b211a240432bc4f51adfdbbdcc03b5ce60a238a84/prek-0.3.10-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:f2fe5ca3fe797a647243c10c0a9f9c2c27c1d4a0cdcb7f836a0cb8c5c2c603d8", size = 5431581, upload-time = "2026-04-21T11:29:20.399Z" }, - { url = "https://files.pythonhosted.org/packages/7d/0c/0dd0013c6a9c6b23d070a57fe5c015c40c7b3e84217e4fb1ca9e465768c6/prek-0.3.10-py3-none-musllinux_1_1_armv7l.whl", hash = "sha256:6d39031e4acf7670b905d77ea807e2ce1b6309604d54740f3152d601b71ae5e0", size = 5318252, upload-time = "2026-04-21T11:29:40.34Z" }, - { url = "https://files.pythonhosted.org/packages/74/d7/5407ca904a7f2ffe63b1333e503d2f21ec23eda93e93fb45a491d78e0d76/prek-0.3.10-py3-none-musllinux_1_1_i686.whl", hash = "sha256:27865f3134dd566ffd76388a9f0de6ee931336794e32741b8e22c90a61f3fc39", size = 5589312, upload-time = "2026-04-21T11:29:18.3Z" }, - { url = "https://files.pythonhosted.org/packages/e0/05/460b0c498db65e9a2e590b6918aad3395291d8d9f2c080fce7201d12a5de/prek-0.3.10-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:3e6b65338a06add2bc47a86f837607b4705855ade9457f2f50676a7a5a509cd9", size = 6122323, upload-time = "2026-04-21T11:29:12.6Z" }, - { url = "https://files.pythonhosted.org/packages/ca/63/fd715bcd9ed5debd60dc589d6117ab9eaf0ef0b106e0bb976e1bcaa96e29/prek-0.3.10-py3-none-win32.whl", hash = "sha256:e2529515ce81292c938181702dbfc75516efa5bb1d4dc1295e6f8b2655eec881", size = 5115503, upload-time = "2026-04-21T11:29:35.677Z" }, - { url = "https://files.pythonhosted.org/packages/6a/0d/a34f57ca3efa228fa75a976fcf698d5705b5b9a750c8dd1b86857b24f3b0/prek-0.3.10-py3-none-win_amd64.whl", hash = "sha256:0846228fa277a8bdf5a3eeea90eb3ac8eff2bf71900919373f12d142e19c8f8c", size = 5496962, upload-time = "2026-04-21T11:29:16.71Z" }, - { url = "https://files.pythonhosted.org/packages/ea/de/8b9f20f712c21756e52feed95fdacd52f48811a7bc6e3816cd04725b1240/prek-0.3.10-py3-none-win_arm64.whl", hash = "sha256:dedcdb4e5a52ee3e0463c061cabdcb94a7443875053cebfadb8228b3bf917035", size = 5340403, upload-time = "2026-04-21T11:29:22.11Z" }, + { url = "https://files.pythonhosted.org/packages/ee/2a/3392fa7d1fd1ce538915baa7597e7203bbe888367a8b15bfd51ca74d4714/prek-0.3.11-py3-none-linux_armv6l.whl", hash = "sha256:787e605716cfdc86ec01e7c5cf62799f39c28d49de5e37d75595c8e6248cb0f3", size = 5423112, upload-time = "2026-04-27T04:22:52.659Z" }, + { url = "https://files.pythonhosted.org/packages/a9/b0/3fc653b30b70d6c2714fc56bcfe1c2439437fc38f60b72bc300603ace4cd/prek-0.3.11-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ef1f37187ca52d75ba9c46b53007476c4eab2c3f11bd23defd57a81c62d90442", size = 5801382, upload-time = "2026-04-27T04:23:04.464Z" }, + { url = "https://files.pythonhosted.org/packages/2e/46/39aedc7843c3703f1f43b686622e4f8cd123e03b87a163e5c8f2fbd56cda/prek-0.3.11-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e0d0828a1b50447502ea1be3f5a84da474fdca558cd5d76a1a5205169bb808c7", size = 5370817, upload-time = "2026-04-27T04:22:49.277Z" }, + { url = "https://files.pythonhosted.org/packages/15/83/df5f3aeacbdea96a88c4f06c98d3932469711fed4e3bf5b703dd6507abe7/prek-0.3.11-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:bf49464b526ee36d2130baf60ab9580560bfaa60efd2997328e6d6671e209014", size = 5621405, upload-time = "2026-04-27T04:23:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/8f/f2/e32c9720747a327669863a4f92d05b9e6fadb851e903b0d7310a97c956a4/prek-0.3.11-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac344529f0d34757c7c95f65e66b9f6440a691f826eaf43f503247bd22023558", size = 5339780, upload-time = "2026-04-27T04:22:47.614Z" }, + { url = "https://files.pythonhosted.org/packages/29/2e/0e2f71b63bc2e5372575d5c1574b0666d2f90d30da51ed706a32cbf465a0/prek-0.3.11-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83672d963f249e2f246d3bec97f8fd2e8032e70da0a7d9acb2fa38af76dd82d2", size = 5735277, upload-time = "2026-04-27T04:23:12.437Z" }, + { url = "https://files.pythonhosted.org/packages/09/46/88abf51ac88eeff1ad2fe7d1797ca1fea43eb1ac1ddb8331463ac5b27ed2/prek-0.3.11-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef70957195d2896a30dd849e64f88344df7bb51af9c950cf16bd11519e7424b0", size = 6622420, upload-time = "2026-04-27T04:23:05.982Z" }, + { url = "https://files.pythonhosted.org/packages/5d/b6/592028a45b084a68b76c7edef909c789d1c96b26761388f63659beef7166/prek-0.3.11-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4f0cc07d2cdb5fe2882015d5fdafc9af98b4c560d4caa1ae948caeab4341b79", size = 6020038, upload-time = "2026-04-27T04:22:54.367Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f7/e97f55a1645a2e9becffeee28892ad8bb66cd144dabfa4392ea8e2674bbe/prek-0.3.11-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:d7554b436dae2ec97f4351a46817e3561657244307d1c0915f355b859f4fab71", size = 5622539, upload-time = "2026-04-27T04:23:01.314Z" }, + { url = "https://files.pythonhosted.org/packages/41/e2/f3119eef6b621782ad216a86d449609858ea34c57cf4a40fc6dc80556d7e/prek-0.3.11-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:caba5d635a5b64b7ac64d903f29b043ca5b0d9d9693543a0ef331ade89e6ad3f", size = 5440681, upload-time = "2026-04-27T04:23:07.422Z" }, + { url = "https://files.pythonhosted.org/packages/04/62/22dd4f59a47654faeebe74651182ecc48d436542646cc92723052dfd9a45/prek-0.3.11-py3-none-musllinux_1_1_armv7l.whl", hash = "sha256:c95a63f19dde48e84b70bd63a235670834af15fa4df8b85d8b7894dd5bc419a9", size = 5314773, upload-time = "2026-04-27T04:22:57.912Z" }, + { url = "https://files.pythonhosted.org/packages/bb/94/a8361462acb8d8f5b8505255b95ffbfc2ee0872a79b4e066eb330692f7be/prek-0.3.11-py3-none-musllinux_1_1_i686.whl", hash = "sha256:7353b45f44a386c676fe96ba72a5ee326b676f789339f405cf6f1d69a1707194", size = 5596208, upload-time = "2026-04-27T04:23:09.08Z" }, + { url = "https://files.pythonhosted.org/packages/04/0c/5f065b86bbeb9977074a055d8a05e90c7201f6c4c7032dada61739b5f8cb/prek-0.3.11-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:39f4e86176ccbb70c098df6abbc8e36c1d86cb81281abe92fb79dcd572418214", size = 6132833, upload-time = "2026-04-27T04:22:56.054Z" }, + { url = "https://files.pythonhosted.org/packages/19/0c/8ab0ae140201dcee505f58b60abbe56bd05ac96b821a6866f6f90c4d971f/prek-0.3.11-py3-none-win32.whl", hash = "sha256:35d2361049653a3dcf27227b7f1b340c5c42a12c0e0361c4b785921bfd125839", size = 5120856, upload-time = "2026-04-27T04:23:02.752Z" }, + { url = "https://files.pythonhosted.org/packages/57/05/9844c1125d3714f6f6c7b475884128a4b0c6c3ee0cd208ead44ca8174687/prek-0.3.11-py3-none-win_amd64.whl", hash = "sha256:a387689cd2e182f92dbb681151ee5a04f494fe97e95d6d783875da90b950e6d5", size = 5510916, upload-time = "2026-04-27T04:22:45.704Z" }, + { url = "https://files.pythonhosted.org/packages/ff/13/24b0288c553dc8d61f44c4d0746fe9bb1e1bd29d1e70571658536e4c0f72/prek-0.3.11-py3-none-win_arm64.whl", hash = "sha256:e4a8f900378a6657c7eb2fc4b12fa5c934edf209d0a24544539842479ec16e0b", size = 5345988, upload-time = "2026-04-27T04:22:50.918Z" }, ] [[package]] @@ -4681,9 +4710,12 @@ dev = [ { name = "docutils" }, { name = "fastavro" }, { name = "google-cloud-bigquery" }, + { name = "ipykernel" }, { name = "moto", extra = ["server"] }, { name = "mypy-boto3-dynamodb" }, { name = "mypy-boto3-glue" }, + { name = "nbformat" }, + { name = "papermill" }, { name = "prek" }, { name = "protobuf" }, { name = "pyarrow-stubs" }, @@ -4771,9 +4803,12 @@ dev = [ { name = "docutils", specifier = "!=0.21.post1" }, { name = "fastavro", specifier = "==1.12.2" }, { name = "google-cloud-bigquery", specifier = ">=3.33.0,<4" }, + { name = "ipykernel", specifier = ">=6.29.0" }, { name = "moto", extras = ["server"], specifier = ">=5.0.2,<6" }, { name = "mypy-boto3-dynamodb", specifier = ">=1.28.18" }, { name = "mypy-boto3-glue", specifier = ">=1.28.18" }, + { name = "nbformat", specifier = ">=5.10.0" }, + { name = "papermill", specifier = ">=2.6.0" }, { name = "prek", specifier = ">=0.2.1,<0.4" }, { name = "protobuf", specifier = "==6.33.5" }, { name = "pyarrow-stubs", specifier = ">=20.0.0.20251107" },