Merge branch 'main' of https://github.com/apache/datafusion into prune_parquet

Author: athlcode

.asf.yaml

@@ -87,6 +87,7 @@ github:
           - "check Cargo.toml formatting"
           - "check configs.md and ***_functions.md is up-to-date"
           - "check example README is up-to-date"
+          - "Check datafusion crates used in datafusion-examples"
           - "Verify MSRV (Min Supported Rust Version)"
     # needs to be updated as part of the release process
     # .asf.yaml doesn't support wildcard branch protection rules, only exact branch names

.github/workflows/release_version_labeler.yaml

@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Add the expected target version as a label to a merged PR.
+# The "target_version" is automatically updated by the dev/update_datafusion_versions.py script.
+
+name: Release Version Labeler
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+on:
+  pull_request_target:
+    types:
+      - closed
+    branches:
+      - main
+
+jobs:
+  label:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-slim
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Add the target release version 
+        uses: actions/github-script@v9.0.0
+        with:
+          script: |
+            const labeler = require('./ci/scripts/release_version_labeler.js');
+            await labeler({ github, context });

.github/workflows/rust.yml

@@ -897,6 +897,16 @@ jobs:
         run: |
           bash ci/scripts/check_examples_docs.sh
 
+  # Ensures that the examples only use the main `datafusion` crate and not its subcrates, which are
+  # already exported by `datafusion` (there are a few exceptions, like `datafusion-proto`).
+  # See: https://github.com/apache/datafusion/pull/14305
+  examples-check-datafusion-crates:
+    name: Check datafusion crates used in datafusion-examples
+    runs-on: ubuntu-slim
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+      - run: python3 ci/scripts/check_examples_datafusion_crates.py
+
   # Verify MSRV for the crates which are directly used by other projects:
   # - datafusion
   # - datafusion-substrait

Cargo.lock

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Checks if datafusion-examples uses `datafusion-...` subcrates unnecessarily, as they are already
+# exported by the `datafusion` crate.
+
+import tomllib
+import sys
+
+allowed_datafusion_crates = {
+    "datafusion",
+    "datafusion-proto",
+    "datafusion-substrait",
+}
+
+crates = set()
+with open("datafusion-examples/Cargo.toml", "rb") as f:
+    cargo = tomllib.load(f)
+    for section in ("dependencies", "dev-dependencies", "build-dependencies"):
+        crates.update(
+            value.get("package", name) if isinstance(value, dict) else name
+            for name, value in cargo.get(section, {}).items()
+        )
+
+crates = [
+    crate for crate in crates if "datafusion" in crate and crate not in allowed_datafusion_crates
+]
+
+if len(crates) > 0:
+    print(
+        "datafusion-examples should only use the main datafusion crate and not its subcrates.",
+        file=sys.stderr,
+    )
+    print(
+        f"Please remove the following crates and use `datafusion::...` instead: {crates}.",
+        file=sys.stderr,
+    )
+
+    sys.exit(1)

ci/scripts/check_examples_datafusion_crates.py

@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Used by the release_version_labeler workflow
+
+module.exports = async ({ github, context }) => {
+    const target_version = 'v56.0.0';
+
+    // create label first if not exists
+    try {
+        await github.rest.issues.getLabel({
+        owner: context.repo.owner,
+        repo: context.repo.repo,
+        name: target_version
+        });
+    } catch (error) {
+        if (error.status === 404) {
+        await github.rest.issues.createLabel({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            name: target_version,
+            color: '222222'
+        });
+        } else {
+        throw error;
+        }
+    }
+
+    await github.rest.issues.addLabels({
+        owner: context.repo.owner,
+        repo: context.repo.repo,
+        issue_number: context.payload.pull_request.number,
+        labels: [target_version]
+    });
+};

ci/scripts/release_version_labeler.js

@@ -38,23 +38,19 @@ workspace = true
 [dependencies]
 arrow = { workspace = true }
 arrow-schema = { workspace = true }
+# note: only use the main datafusion crate for examples
 datafusion = { workspace = true, default-features = true, features = ["parquet_encryption"] }
-datafusion-common = { workspace = true }
 nom = "8.0.0"
 tempfile = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot", "fs"] }
 
 [dev-dependencies]
 arrow-flight = { workspace = true }
 async-trait = { workspace = true }
+base64 = "0.23.0"
 bytes = { workspace = true }
 dashmap = { workspace = true }
-# note only use main datafusion crate for examples
-base64 = "0.23.0"
-datafusion-expr = { workspace = true }
-datafusion-physical-expr-adapter = { workspace = true }
 datafusion-proto = { workspace = true, features = ["parquet"] }
-datafusion-sql = { workspace = true }
 env_logger = { workspace = true }
 futures = { workspace = true }
 insta = { workspace = true }

datafusion-examples/Cargo.toml

@@ -50,11 +50,11 @@ use datafusion::execution::context::SessionContext;
 use datafusion::execution::object_store::ObjectStoreUrl;
 use datafusion::parquet::arrow::ArrowWriter;
 use datafusion::physical_expr::PhysicalExpr;
-use datafusion::physical_plan::ExecutionPlan;
-use datafusion::prelude::SessionConfig;
-use datafusion_physical_expr_adapter::{
+use datafusion::physical_expr_adapter::{
     DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,
 };
+use datafusion::physical_plan::ExecutionPlan;
+use datafusion::prelude::SessionConfig;
 use datafusion_proto::bytes::{
     physical_plan_from_bytes_with_proto_converter,
     physical_plan_to_bytes_with_proto_converter,

datafusion-examples/examples/custom_data_source/adapter_serialization.rs

@@ -34,10 +34,10 @@ use datafusion::execution::object_store::ObjectStoreUrl;
 use datafusion::parquet::arrow::ArrowWriter;
 use datafusion::physical_expr::PhysicalExpr;
 use datafusion::physical_expr::expressions::CastExpr;
-use datafusion::prelude::SessionConfig;
-use datafusion_physical_expr_adapter::{
+use datafusion::physical_expr_adapter::{
     DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,
 };
+use datafusion::prelude::SessionConfig;
 use object_store::memory::InMemory;
 use object_store::path::Path;
 use object_store::{ObjectStore, ObjectStoreExt, PutPayload};

datafusion-examples/examples/custom_data_source/custom_file_casts.rs

@@ -38,12 +38,12 @@ use datafusion::logical_expr::{Expr, TableProviderFilterPushDown, TableType};
 use datafusion::parquet::arrow::ArrowWriter;
 use datafusion::parquet::file::properties::WriterProperties;
 use datafusion::physical_expr::PhysicalExpr;
-use datafusion::physical_plan::ExecutionPlan;
-use datafusion::prelude::{SessionConfig, lit};
-use datafusion_physical_expr_adapter::{
+use datafusion::physical_expr_adapter::{
     DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,
     replace_columns_with_literals,
 };
+use datafusion::physical_plan::ExecutionPlan;
+use datafusion::prelude::{SessionConfig, lit};
 use futures::StreamExt;
 use object_store::memory::InMemory;
 use object_store::path::Path;