From f1d225738f9d893fca6e0fae1b88e0e258ee4545 Mon Sep 17 00:00:00 2001 From: Alex Kasko Date: Tue, 2 Jun 2026 14:03:27 +0100 Subject: [PATCH] Optimizer and pushdown common code --- .github/workflows/ConnectorTests.yml | 72 +++- Makefile | 9 + src/include/dbconnector/bind_data.hpp | 17 + .../optimizer/aggregate_bind_data.hpp | 16 + .../optimizer/aggregate_optimizer.hpp | 32 ++ .../dbconnector/optimizer/optimizer_util.hpp | 34 ++ .../order_by_and_limit_bind_data.hpp | 14 + .../order_by_and_limit_optimizer.hpp | 27 ++ .../dbconnector/query/query_writer.hpp | 23 ++ .../table_scan/filter_pushdown.hpp | 31 ++ .../dbconnector/table_scan/filter_util.hpp | 21 ++ .../table_scan/table_scan_exception.hpp | 24 ++ src/optimizer/aggregate_optimizer.cpp | 330 ++++++++++++++++ src/optimizer/optimizer_util.cpp | 77 ++++ .../order_by_and_limit_optimizer.cpp | 356 ++++++++++++++++++ src/query/query_writer.cpp | 66 ++++ src/table_scan/filter_pushdown.cpp | 166 ++++++++ src/table_scan/filter_util.cpp | 26 ++ test/extension/CMakeLists.txt | 30 ++ test/extension/test_query.cpp | 10 + 20 files changed, 1376 insertions(+), 5 deletions(-) create mode 100644 src/include/dbconnector/bind_data.hpp create mode 100644 src/include/dbconnector/optimizer/aggregate_bind_data.hpp create mode 100644 src/include/dbconnector/optimizer/aggregate_optimizer.hpp create mode 100644 src/include/dbconnector/optimizer/optimizer_util.hpp create mode 100644 src/include/dbconnector/optimizer/order_by_and_limit_bind_data.hpp create mode 100644 src/include/dbconnector/optimizer/order_by_and_limit_optimizer.hpp create mode 100644 src/include/dbconnector/query/query_writer.hpp create mode 100644 src/include/dbconnector/table_scan/filter_pushdown.hpp create mode 100644 src/include/dbconnector/table_scan/filter_util.hpp create mode 100644 src/include/dbconnector/table_scan/table_scan_exception.hpp create mode 100644 src/optimizer/aggregate_optimizer.cpp create mode 100644 src/optimizer/optimizer_util.cpp create mode 100644 src/optimizer/order_by_and_limit_optimizer.cpp create mode 100644 src/query/query_writer.cpp create mode 100644 src/table_scan/filter_pushdown.cpp create mode 100644 src/table_scan/filter_util.cpp create mode 100644 test/extension/CMakeLists.txt create mode 100644 test/extension/test_query.cpp diff --git a/.github/workflows/ConnectorTests.yml b/.github/workflows/ConnectorTests.yml index f8b52cd..1da3074 100644 --- a/.github/workflows/ConnectorTests.yml +++ b/.github/workflows/ConnectorTests.yml @@ -9,7 +9,7 @@ jobs: name: Format Check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - run: | python -m pip install --user clang_format==11.0.1 make format-check @@ -22,7 +22,7 @@ jobs: GEN: ninja steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Dependencies run: | @@ -41,7 +41,7 @@ jobs: runs-on: windows-latest needs: linux-tests-amd64 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Build and Test shell: cmd @@ -59,7 +59,7 @@ jobs: env: GEN: ninja steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Build and Test run: | @@ -73,7 +73,7 @@ jobs: GEN: ninja steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Dependencies run: | @@ -86,3 +86,65 @@ jobs: - name: Build and Test run: | make test_threads + + linux-ext-amd64: + name: Linux Extension (amd64) + runs-on: ubuntu-latest + needs: linux-tests-amd64 + env: + GEN: ninja + CC: 'ccache gcc' + CXX: 'ccache g++' + CCACHE_DIR: ${{ github.workspace }}/ccache + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Checkout Engine + uses: actions/checkout@v6 + with: + path: 'duckdb' + ref: 'main' + repository: duckdb/duckdb + + - name: Dependencies + run: | + echo "set man-db/auto-update false" | sudo debconf-communicate + sudo dpkg-reconfigure man-db + sudo apt-get update -y -q -o=Dpkg::Use-Pty=0 + sudo apt-get install -y -q -o=Dpkg::Use-Pty=0 \ + build-essential \ + ccache \ + cmake \ + ninja-build + + - name: Cache Key + id: cache_key + working-directory: duckdb + run: | + DUCKDB_VERSION=$(git rev-parse --short HEAD) + KEY="${{ runner.os }}-${{ runner.arch }}-$DUCKDB_VERSION" + echo "value=${KEY}" >> "${GITHUB_OUTPUT}" + + - name: Restore Cache + uses: actions/cache/restore@v5 + with: + path: ${{ github.workspace }}/ccache + key: ${{ steps.cache_key.outputs.value }} + + - name: Build Engine + working-directory: duckdb + run: | + make release + + - name: Save Cache + uses: actions/cache/save@v5 + with: + path: ${{ github.workspace }}/ccache + key: ${{ steps.cache_key.outputs.value }} + + - name: Build and Test Extension + env: + DUCKDB_SRC_PATH: ${{ github.workspace }}/duckdb + run: | + make test_ext diff --git a/Makefile b/Makefile index be6027c..db99715 100644 --- a/Makefile +++ b/Makefile @@ -30,8 +30,17 @@ build_threads: cmake -DCMAKE_BUILD_TYPE=Debug $(GENERATOR) -DENABLE_THREAD_SANITIZER=ON ../../test && \ cmake --build . --config Debug +build_ext: + mkdir -p build/debug + cd build/debug && \ + cmake -DCMAKE_BUILD_TYPE=Debug $(GENERATOR) ../../test/extension && \ + cmake --build . --config Debug + test: build build/debug/test_database_connector test_threads: build_threads build/debug/test_database_connector + +test_ext: build_ext + build/debug/test_database_connector_ext diff --git a/src/include/dbconnector/bind_data.hpp b/src/include/dbconnector/bind_data.hpp new file mode 100644 index 0000000..4e81dfd --- /dev/null +++ b/src/include/dbconnector/bind_data.hpp @@ -0,0 +1,17 @@ +#pragma once + +#include "duckdb/function/function.hpp" + +#include "dbconnector/optimizer/aggregate_bind_data.hpp" +#include "dbconnector/optimizer/order_by_and_limit_bind_data.hpp" + +namespace dbconnector { + +class BindData : public duckdb::FunctionData { +public: + virtual optimizer::AggregateBindData &GetAggregateBindData() = 0; + + virtual optimizer::OrderByAndLimitBindData &GetOrderByAndLimitBindData() = 0; +}; + +} // namespace dbconnector diff --git a/src/include/dbconnector/optimizer/aggregate_bind_data.hpp b/src/include/dbconnector/optimizer/aggregate_bind_data.hpp new file mode 100644 index 0000000..7528650 --- /dev/null +++ b/src/include/dbconnector/optimizer/aggregate_bind_data.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include + +namespace dbconnector { +namespace optimizer { + +struct AggregateBindData { + std::string aggregate_select_list; + std::string group_by_clause; + std::string aggregate_where_clause; + bool has_aggregate_pushdown = false; +}; + +} // namespace optimizer +} // namespace dbconnector diff --git a/src/include/dbconnector/optimizer/aggregate_optimizer.hpp b/src/include/dbconnector/optimizer/aggregate_optimizer.hpp new file mode 100644 index 0000000..df96bc0 --- /dev/null +++ b/src/include/dbconnector/optimizer/aggregate_optimizer.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include + +#include "duckdb/main/client_context.hpp" +#include "duckdb/optimizer/optimizer_extension.hpp" + +namespace dbconnector { +namespace optimizer { + +typedef bool (*should_push_aggregate_t)(duckdb::ClientContext &context, duckdb::LogicalAggregate &aggr); + +class AggregateOptimizer { +public: + struct Config { + bool enabled = false; + char identifier_quote = '"'; + std::string table_scan_name; + should_push_aggregate_t should_push_aggregate = nullptr; + }; + + static Config CreateConfig(duckdb::ClientContext &ctx, const std::string &enabled_option, + + char identifier_quote, std::string table_scan_name, + should_push_aggregate_t should_push_aggregate = nullptr); + + static void Optimize(const Config &config, duckdb::OptimizerExtensionInput &input, + duckdb::unique_ptr &op); +}; + +} // namespace optimizer +} // namespace dbconnector diff --git a/src/include/dbconnector/optimizer/optimizer_util.hpp b/src/include/dbconnector/optimizer/optimizer_util.hpp new file mode 100644 index 0000000..167e097 --- /dev/null +++ b/src/include/dbconnector/optimizer/optimizer_util.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include + +#include "duckdb/common/types.hpp" +#include "duckdb/planner/logical_operator.hpp" +#include "duckdb/planner/operator/logical_get.hpp" +#include "duckdb/planner/column_binding.hpp" + +#include "dbconnector/bind_data.hpp" + +namespace dbconnector { +namespace optimizer { + +struct TracedBindingColumn { + std::string col_name; + duckdb::LogicalType col_type; + + bool Found() { + return !col_name.empty(); + } +}; + +class OptimizerUtil { +public: + static bool FindExtensionGet(const std::string &table_scan_name, duckdb::LogicalOperator &start, + duckdb::LogicalGet *&get_out, dbconnector::BindData *&bind_out); + + static TracedBindingColumn TraceBindingToColumn(duckdb::ColumnBinding binding, duckdb::LogicalOperator &child, + duckdb::LogicalGet &get); +}; + +} // namespace optimizer +} // namespace dbconnector diff --git a/src/include/dbconnector/optimizer/order_by_and_limit_bind_data.hpp b/src/include/dbconnector/optimizer/order_by_and_limit_bind_data.hpp new file mode 100644 index 0000000..5fd3191 --- /dev/null +++ b/src/include/dbconnector/optimizer/order_by_and_limit_bind_data.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include + +namespace dbconnector { +namespace optimizer { + +struct OrderByAndLimitBindData { + std::string limit_clause; + std::string order_by_clause; +}; + +} // namespace optimizer +} // namespace dbconnector diff --git a/src/include/dbconnector/optimizer/order_by_and_limit_optimizer.hpp b/src/include/dbconnector/optimizer/order_by_and_limit_optimizer.hpp new file mode 100644 index 0000000..b963f52 --- /dev/null +++ b/src/include/dbconnector/optimizer/order_by_and_limit_optimizer.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include + +#include "duckdb/main/client_context.hpp" +#include "duckdb/optimizer/optimizer_extension.hpp" + +namespace dbconnector { +namespace optimizer { + +class OrderByAndLimitOptimizer { +public: + struct Config { + bool enabled = false; + char identifier_quote = '"'; + std::string table_scan_name; + }; + + static Config CreateConfig(duckdb::ClientContext &ctx, const std::string &enabled_option, char identifier_quote, + std::string table_scan_name); + + static void Optimize(const Config &config, duckdb::OptimizerExtensionInput &input, + duckdb::unique_ptr &op); +}; + +} // namespace optimizer +} // namespace dbconnector diff --git a/src/include/dbconnector/query/query_writer.hpp b/src/include/dbconnector/query/query_writer.hpp new file mode 100644 index 0000000..7c6cf76 --- /dev/null +++ b/src/include/dbconnector/query/query_writer.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include + +#include "duckdb/common/types/value.hpp" + +namespace dbconnector { +namespace query { + +class QueryWriter { +public: + static std::string WriteIdentifier(const std::string &identifier, char identifier_quote); + static std::string WriteLiteral(const std::string &identifier); + static std::string WriteConstant(const duckdb::Value &val); + +private: + static std::string EncodeBlob(const std::string &val); + static std::string EscapeQuotes(const std::string &text, char quote); + static std::string WriteQuoted(const std::string &text, char quote); +}; + +} // namespace query +} // namespace dbconnector diff --git a/src/include/dbconnector/table_scan/filter_pushdown.hpp b/src/include/dbconnector/table_scan/filter_pushdown.hpp new file mode 100644 index 0000000..72c8d9e --- /dev/null +++ b/src/include/dbconnector/table_scan/filter_pushdown.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include + +#include "duckdb/common/enums/expression_type.hpp" +#include "duckdb/planner/table_filter.hpp" + +namespace dbconnector { +namespace table_scan { + +class FilterPushdown { + struct Config { + char identifier_quote = '"'; + }; + +public: + static Config CreateConfig(char identifier_quote); + + static std::string TransformFilter(const Config &config, const std::string &column_name, + const duckdb::TableFilter &filter); + +private: + static std::string TransformExpression(const std::string &column_name, const duckdb::Expression &expr); + static std::string TransformComparison(duckdb::ExpressionType type); + static std::string CreateExpression(const std::string &column_name, + const duckdb::vector> &filters, + const std::string &op); +}; + +} // namespace table_scan +} // namespace dbconnector diff --git a/src/include/dbconnector/table_scan/filter_util.hpp b/src/include/dbconnector/table_scan/filter_util.hpp new file mode 100644 index 0000000..e28843d --- /dev/null +++ b/src/include/dbconnector/table_scan/filter_util.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include + +#include "duckdb/common/enums/expression_type.hpp" +#include "duckdb/planner/table_filter.hpp" + +namespace dbconnector { +namespace table_scan { + +class FilterUtil { +public: + static const duckdb::Expression &GetExpression(const duckdb::TableFilter &filter, const std::string &call_context); + + static bool IsInternalFilter(const duckdb::TableFilter &filter); + + static std::string ToString(const duckdb::TableFilter &filter); +}; + +} // namespace table_scan +} // namespace dbconnector diff --git a/src/include/dbconnector/table_scan/table_scan_exception.hpp b/src/include/dbconnector/table_scan/table_scan_exception.hpp new file mode 100644 index 0000000..eeca5b1 --- /dev/null +++ b/src/include/dbconnector/table_scan/table_scan_exception.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include + +namespace dbconnector { +namespace table_scan { + +class TableScanException : public std::exception { +protected: + std::string message; + +public: + TableScanException() = default; + + TableScanException(const std::string &message) : message(message.data(), message.length()) { + } + + virtual const char *what() const noexcept { + return message.c_str(); + } +}; + +} // namespace table_scan +} // namespace dbconnector diff --git a/src/optimizer/aggregate_optimizer.cpp b/src/optimizer/aggregate_optimizer.cpp new file mode 100644 index 0000000..9728f78 --- /dev/null +++ b/src/optimizer/aggregate_optimizer.cpp @@ -0,0 +1,330 @@ +#include "dbconnector/optimizer/aggregate_optimizer.hpp" + +#include "duckdb/common/types/value.hpp" +#include "duckdb/planner/operator/logical_aggregate.hpp" +#include "duckdb/planner/operator/logical_comparison_join.hpp" +#include "duckdb/planner/operator/logical_get.hpp" +#include "duckdb/planner/operator/logical_limit.hpp" +#include "duckdb/planner/operator/logical_order.hpp" +#include "duckdb/planner/operator/logical_top_n.hpp" +#include "duckdb/planner/operator/logical_projection.hpp" +#include "duckdb/planner/expression/bound_aggregate_expression.hpp" +#include "duckdb/planner/expression/bound_columnref_expression.hpp" +#include "duckdb/planner/expression_iterator.hpp" + +#include "dbconnector/bind_data.hpp" +#include "dbconnector/optimizer/aggregate_bind_data.hpp" +#include "dbconnector/optimizer/optimizer_util.hpp" +#include "dbconnector/query/query_writer.hpp" +#include "dbconnector/table_scan/filter_pushdown.hpp" + +namespace dbconnector { +namespace optimizer { + +using namespace duckdb; + +AggregateOptimizer::Config AggregateOptimizer::CreateConfig(ClientContext &ctx, const std::string &enabled_option, + + char identifier_quote, std::string table_scan_name, + should_push_aggregate_t should_push_aggregate) { + Config res; + + res.enabled = false; + Value enabled_val; + if (ctx.TryGetCurrentSetting(enabled_option, enabled_val) && !enabled_val.IsNull()) { + res.enabled = BooleanValue::Get(enabled_val); + } + + res.identifier_quote = identifier_quote; + res.table_scan_name = std::move(table_scan_name); + res.should_push_aggregate = should_push_aggregate; + + return res; +} + +static const unordered_set PUSHABLE_AGGREGATES = {"count_star", "count", "sum", "avg", "min", "max"}; + +static bool CanPushAggregate(LogicalAggregate &aggr) { + if (aggr.grouping_sets.size() > 1) { + return false; + } + if (!aggr.grouping_functions.empty()) { + return false; + } + for (auto &group : aggr.groups) { + if (group->GetExpressionClass() != ExpressionClass::BOUND_COLUMN_REF) { + return false; + } + } + for (auto &expr : aggr.expressions) { + if (expr->GetExpressionClass() != ExpressionClass::BOUND_AGGREGATE) { + return false; + } + auto &agg_expr = expr->Cast(); + if (agg_expr.IsDistinct()) { + return false; + } + if (agg_expr.filter) { + return false; + } + if (agg_expr.order_bys && !agg_expr.order_bys->orders.empty()) { + return false; + } + if (PUSHABLE_AGGREGATES.find(agg_expr.function.GetName()) == PUSHABLE_AGGREGATES.end()) { + return false; + } + if (agg_expr.function.GetName() != "count_star") { + if (agg_expr.children.size() != 1) { + return false; + } + if (agg_expr.children[0]->GetExpressionClass() != ExpressionClass::BOUND_COLUMN_REF) { + return false; + } + } + } + return true; +} + +struct PushedAggregate { + string select_list; + string group_by_clause; + string where_clause; + bool success = false; + + bool PushedDown() { + return success; + } +}; + +static PushedAggregate TryPushAggregateToMySQL(const AggregateOptimizer::Config &config, LogicalAggregate &aggr, + LogicalOperator &aggr_child, LogicalGet &get) { + PushedAggregate res; + if (!CanPushAggregate(aggr)) { + return res; + } + + vector select_fragments; + vector group_names; + vector new_types; + vector new_names; + + for (auto &group : aggr.groups) { + auto &col_ref = group->Cast(); + TracedBindingColumn traced_binding = OptimizerUtil::TraceBindingToColumn(col_ref.binding, aggr_child, get); + if (!traced_binding.Found()) { + return res; + } + string quoted = + dbconnector::query::QueryWriter::WriteIdentifier(traced_binding.col_name, config.identifier_quote); + select_fragments.push_back(quoted); + group_names.push_back(quoted); + new_types.push_back(traced_binding.col_type); + new_names.push_back(traced_binding.col_name); + } + + idx_t agg_idx = 0; + for (auto &expr : aggr.expressions) { + auto &agg_expr = expr->Cast(); + string fragment; + string alias = "_agg_" + to_string(agg_idx); + + if (agg_expr.function.GetName() == "count_star") { + fragment = + "COUNT(*) AS " + dbconnector::query::QueryWriter::WriteIdentifier(alias, config.identifier_quote); + } else { + auto &child_ref = agg_expr.children[0]->Cast(); + TracedBindingColumn traced_binding = + OptimizerUtil::TraceBindingToColumn(child_ref.binding, aggr_child, get); + if (!traced_binding.Found()) { + return res; + } + string quoted_col = + dbconnector::query::QueryWriter::WriteIdentifier(traced_binding.col_name, config.identifier_quote); + string func_upper; + if (agg_expr.function.GetName() == "count") { + func_upper = "COUNT"; + } else if (agg_expr.function.GetName() == "sum") { + func_upper = "SUM"; + } else if (agg_expr.function.GetName() == "avg") { + func_upper = "AVG"; + } else if (agg_expr.function.GetName() == "min") { + func_upper = "MIN"; + } else if (agg_expr.function.GetName() == "max") { + func_upper = "MAX"; + } else { + return res; + } + if (func_upper == "SUM" && traced_binding.col_type.id() == LogicalTypeId::DECIMAL) { + return res; + } + if (func_upper == "AVG" && traced_binding.col_type.id() == LogicalTypeId::DECIMAL) { + return res; + } + string agg_sql = func_upper + "(" + quoted_col + ")"; + if (func_upper == "AVG") { + agg_sql = "CAST(" + agg_sql + " AS DOUBLE)"; + } + fragment = + agg_sql + " AS " + dbconnector::query::QueryWriter::WriteIdentifier(alias, config.identifier_quote); + } + + select_fragments.push_back(fragment); + new_types.push_back(agg_expr.GetReturnType()); + new_names.push_back(alias); + agg_idx++; + } + + res.select_list = StringUtil::Join(select_fragments, ", "); + + if (!group_names.empty()) { + res.group_by_clause = " GROUP BY " + StringUtil::Join(group_names, ", "); + } + + if (get.table_filters.HasFilters()) { + string where_clause; + for (auto &entry : get.table_filters) { + ProjectionIndex proj_idx = entry.GetIndex(); + ColumnIndex col_idx = get.GetColumnIndex(proj_idx); + column_t table_col_idx = col_idx.GetPrimaryIndex(); + if (table_col_idx >= get.names.size()) { + return res; + } + auto column_name = get.names[table_col_idx]; + auto config = dbconnector::table_scan::FilterPushdown::CreateConfig('`'); + auto new_filter = + dbconnector::table_scan::FilterPushdown::TransformFilter(config, column_name, entry.Filter()); + if (new_filter.empty()) { + return res; + } + if (!where_clause.empty()) { + where_clause += " AND "; + } + where_clause += new_filter; + } + if (!where_clause.empty()) { + res.where_clause = where_clause; + } + } + + get.returned_types = new_types; + get.names = new_names; + vector new_column_ids; + for (idx_t i = 0; i < new_types.size(); i++) { + new_column_ids.push_back(ColumnIndex(i)); + } + get.SetColumnIds(std::move(new_column_ids)); + get.projection_ids.clear(); + get.table_filters.ClearFilters(); + + res.success = true; + return res; +} + +struct AggregateRewriteInfo { + TableIndex group_index; + TableIndex aggregate_index; + TableIndex scan_table_index; + idx_t num_groups; +}; + +static void RewriteExpression(unique_ptr &expr, AggregateRewriteInfo &info) { + if (expr->GetExpressionClass() == ExpressionClass::BOUND_COLUMN_REF) { + auto &col_ref = expr->Cast(); + if (col_ref.depth > 0) { + return; + } + if (col_ref.binding.table_index == info.group_index) { + col_ref.binding.table_index = info.scan_table_index; + } else if (col_ref.binding.table_index == info.aggregate_index) { + col_ref.binding.table_index = info.scan_table_index; + col_ref.binding.column_index = ProjectionIndex(col_ref.binding.column_index.GetIndex() + info.num_groups); + } + } + ExpressionIterator::EnumerateChildren(*expr, + [&](unique_ptr &child) { RewriteExpression(child, info); }); +} + +static void RewriteBindingsInOperator(LogicalOperator &op, AggregateRewriteInfo &info) { + for (auto &expr : op.expressions) { + RewriteExpression(expr, info); + } + if (op.type == LogicalOperatorType::LOGICAL_ORDER_BY) { + auto &order = op.Cast(); + for (auto &node : order.orders) { + RewriteExpression(node.expression, info); + } + } + if (op.type == LogicalOperatorType::LOGICAL_TOP_N) { + auto &topn = op.Cast(); + for (auto &node : topn.orders) { + RewriteExpression(node.expression, info); + } + } + if (op.type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) { + auto &join = op.Cast(); + for (auto &cond : join.conditions) { + RewriteExpression(cond.LeftReference(), info); + RewriteExpression(cond.RightReference(), info); + } + } +} + +static void RewriteBindingsInTree(LogicalOperator &op, AggregateRewriteInfo &info) { + RewriteBindingsInOperator(op, info); + for (auto &child : op.children) { + RewriteBindingsInTree(*child, info); + } +} + +static void OptimizeAggregates(const AggregateOptimizer::Config &config, ClientContext &context, + unique_ptr &op, vector &rewrites) { + if (!config.enabled) { + return; + } + + for (idx_t i = 0; i < op->children.size(); i++) { + auto &child = op->children[i]; + if (child->type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) { + auto &aggr = child->Cast(); + LogicalGet *get = nullptr; + dbconnector::BindData *bind_data = nullptr; + if (!aggr.children.empty() && + OptimizerUtil::FindExtensionGet(config.table_scan_name, *aggr.children[0], get, bind_data)) { + if (config.should_push_aggregate && !config.should_push_aggregate(context, aggr)) { + OptimizeAggregates(config, context, child, rewrites); + continue; + } + + PushedAggregate pushed_aggr = TryPushAggregateToMySQL(config, aggr, *aggr.children[0], *get); + auto &aggr_bind_data = bind_data->GetAggregateBindData(); + aggr_bind_data.aggregate_select_list = pushed_aggr.select_list; + aggr_bind_data.group_by_clause = pushed_aggr.group_by_clause; + aggr_bind_data.aggregate_where_clause = pushed_aggr.where_clause; + aggr_bind_data.has_aggregate_pushdown = pushed_aggr.PushedDown(); + if (pushed_aggr.PushedDown()) { + AggregateRewriteInfo info; + info.group_index = aggr.group_index; + info.aggregate_index = aggr.aggregate_index; + info.scan_table_index = get->table_index; + info.num_groups = aggr.groups.size(); + rewrites.push_back(info); + op->children[i] = std::move(aggr.children[0]); + continue; + } + } + } + OptimizeAggregates(config, context, child, rewrites); + } +} + +void AggregateOptimizer::Optimize(const AggregateOptimizer::Config &config, OptimizerExtensionInput &input, + unique_ptr &op) { + vector rewrites; + OptimizeAggregates(config, input.context, op, rewrites); + for (auto &info : rewrites) { + RewriteBindingsInTree(*op, info); + } +} + +} // namespace optimizer +} // namespace dbconnector diff --git a/src/optimizer/optimizer_util.cpp b/src/optimizer/optimizer_util.cpp new file mode 100644 index 0000000..a310f59 --- /dev/null +++ b/src/optimizer/optimizer_util.cpp @@ -0,0 +1,77 @@ +#include "dbconnector/optimizer/optimizer_util.hpp" + +#include "duckdb/planner/expression/bound_columnref_expression.hpp" +#include "duckdb/planner/operator/logical_projection.hpp" + +namespace dbconnector { +namespace optimizer { + +using namespace duckdb; + +bool OptimizerUtil::FindExtensionGet(const std::string &table_scan_name, LogicalOperator &start, LogicalGet *&get_out, + dbconnector::BindData *&bind_out) { + reference current = start; + while (current.get().type == LogicalOperatorType::LOGICAL_PROJECTION) { + if (current.get().children.empty()) { + return false; + } + current = *current.get().children[0]; + } + if (current.get().type != LogicalOperatorType::LOGICAL_GET) { + return false; + } + auto &get = current.get().Cast(); + if (get.function.name != table_scan_name) { + return false; + } + get_out = &get; + bind_out = &get.bind_data->Cast(); + return true; +} + +TracedBindingColumn OptimizerUtil::TraceBindingToColumn(ColumnBinding binding, LogicalOperator &child, + LogicalGet &get) { + TracedBindingColumn res; + reference current = child; + while (current.get().type == LogicalOperatorType::LOGICAL_PROJECTION) { + auto &proj = current.get().Cast(); + if (binding.table_index != proj.table_index) { + break; + } + if (binding.column_index >= proj.expressions.size()) { + return res; + } + auto &proj_expr = *proj.expressions[binding.column_index]; + if (proj_expr.GetExpressionClass() != ExpressionClass::BOUND_COLUMN_REF) { + return res; + } + auto &inner_ref = proj_expr.Cast(); + if (inner_ref.depth > 0) { + return res; + } + binding = inner_ref.binding; + current = *current.get().children[0]; + } + + if (binding.table_index != get.table_index) { + return res; + } + auto &column_ids = get.GetColumnIds(); + if (binding.column_index >= column_ids.size()) { + return res; + } + auto &col_index = column_ids[binding.column_index]; + if (col_index.IsRowIdColumn()) { + return res; + } + auto actual_col_idx = col_index.GetPrimaryIndex(); + if (actual_col_idx >= get.names.size()) { + return res; + } + res.col_name = get.names[actual_col_idx]; + res.col_type = get.returned_types[actual_col_idx]; + return res; +} + +} // namespace optimizer +} // namespace dbconnector diff --git a/src/optimizer/order_by_and_limit_optimizer.cpp b/src/optimizer/order_by_and_limit_optimizer.cpp new file mode 100644 index 0000000..590db60 --- /dev/null +++ b/src/optimizer/order_by_and_limit_optimizer.cpp @@ -0,0 +1,356 @@ +#include "dbconnector/optimizer/order_by_and_limit_optimizer.hpp" + +#include "duckdb/common/types/value.hpp" +#include "duckdb/planner/operator/logical_get.hpp" +#include "duckdb/planner/operator/logical_limit.hpp" +#include "duckdb/planner/operator/logical_order.hpp" +#include "duckdb/planner/operator/logical_top_n.hpp" +#include "duckdb/planner/operator/logical_projection.hpp" +#include "duckdb/planner/expression/bound_columnref_expression.hpp" +#include "duckdb/planner/expression_iterator.hpp" + +#include "dbconnector/bind_data.hpp" +#include "dbconnector/optimizer/optimizer_util.hpp" +#include "dbconnector/query/query_writer.hpp" + +namespace dbconnector { +namespace optimizer { + +using namespace duckdb; + +OrderByAndLimitOptimizer::Config OrderByAndLimitOptimizer::CreateConfig(ClientContext &ctx, + const std::string &enabled_option, + char identifier_quote, + std::string table_scan_name) { + Config res; + + res.enabled = false; + Value enabled_val; + if (ctx.TryGetCurrentSetting(enabled_option, enabled_val) && !enabled_val.IsNull()) { + res.enabled = BooleanValue::Get(enabled_val); + } + + res.identifier_quote = identifier_quote; + res.table_scan_name = std::move(table_scan_name); + + return res; +} + +static string TraceColumnToGet(const OrderByAndLimitOptimizer::Config &config, Expression &expr, LogicalOperator &child, + LogicalGet &get) { + if (expr.GetExpressionClass() != ExpressionClass::BOUND_COLUMN_REF) { + return std::string(); + } + auto &col_ref = expr.Cast(); + if (col_ref.depth > 0) { + return std::string(); + } + auto binding = col_ref.binding; + + reference current = child; + while (current.get().type == LogicalOperatorType::LOGICAL_PROJECTION) { + auto &proj = current.get().Cast(); + if (binding.table_index != proj.table_index) { + break; + } + if (binding.column_index >= proj.expressions.size()) { + return std::string(); + } + auto &proj_expr = *proj.expressions[binding.column_index]; + if (proj_expr.GetExpressionClass() != ExpressionClass::BOUND_COLUMN_REF) { + return std::string(); + } + auto &inner_ref = proj_expr.Cast(); + if (inner_ref.depth > 0) { + return std::string(); + } + binding = inner_ref.binding; + current = *current.get().children[0]; + } + + if (binding.table_index != get.table_index) { + return std::string(); + } + auto &column_ids = get.GetColumnIds(); + if (binding.column_index >= column_ids.size()) { + return std::string(); + } + auto &col_index = column_ids[binding.column_index]; + if (col_index.IsRowIdColumn()) { + return std::string(); + } + + auto actual_col_idx = col_index.GetPrimaryIndex(); + if (actual_col_idx >= get.names.size()) { + return std::string(); + } + return dbconnector::query::QueryWriter::WriteIdentifier(get.names[actual_col_idx], config.identifier_quote); +} + +static string TryBuildOrderByClause(const OrderByAndLimitOptimizer::Config &config, vector &orders, + LogicalOperator &child, LogicalGet &get) { + vector fragments; + for (auto &order : orders) { + string col_name = TraceColumnToGet(config, *order.expression, child, get); + if (col_name.empty()) { + return std::string(); + } + + OrderType direction = order.type; + OrderByNullType null_order = order.null_order; + + if (direction == OrderType::ORDER_DEFAULT) { + direction = OrderType::ASCENDING; + } + if (null_order == OrderByNullType::ORDER_DEFAULT) { + null_order = + (direction == OrderType::ASCENDING) ? OrderByNullType::NULLS_LAST : OrderByNullType::NULLS_FIRST; + } + + if (direction == OrderType::ASCENDING) { + if (null_order == OrderByNullType::NULLS_FIRST) { + fragments.push_back(col_name + " ASC"); + } else { + fragments.push_back(col_name + " IS NULL, " + col_name + " ASC"); + } + } else { + if (null_order == OrderByNullType::NULLS_FIRST) { + fragments.push_back(col_name + " IS NOT NULL, " + col_name + " DESC"); + } else { + fragments.push_back(col_name + " DESC"); + } + } + } + return " ORDER BY " + StringUtil::Join(fragments, ", "); +} + +static void CollectBindingRefs(Expression &expr, idx_t target_table_index, unordered_set &referenced) { + if (expr.GetExpressionClass() == ExpressionClass::BOUND_COLUMN_REF) { + auto &ref = expr.Cast(); + if (ref.binding.table_index.index == target_table_index) { + referenced.insert(ref.binding.column_index); + } + } + ExpressionIterator::EnumerateChildren( + expr, [&](unique_ptr &child) { CollectBindingRefs(*child, target_table_index, referenced); }); +} + +static void RewriteBindingRefs(Expression &expr, idx_t target_table_index, unordered_map &old_to_new) { + if (expr.GetExpressionClass() == ExpressionClass::BOUND_COLUMN_REF) { + auto &ref = expr.Cast(); + if (ref.binding.table_index.index == target_table_index) { + auto it = old_to_new.find(ref.binding.column_index); + if (it != old_to_new.end()) { + ref.binding.column_index = ProjectionIndex(it->second); + } + } + } + ExpressionIterator::EnumerateChildren( + expr, [&](unique_ptr &child) { RewriteBindingRefs(*child, target_table_index, old_to_new); }); +} + +static void PruneProjectionLayer(LogicalProjection &proj, const unordered_set &keep_indices, + vector &above_projs) { + vector> new_exprs; + unordered_map old_to_new; + for (idx_t i = 0; i < proj.expressions.size(); i++) { + if (keep_indices.count(i)) { + old_to_new[i] = new_exprs.size(); + new_exprs.push_back(std::move(proj.expressions[i])); + } + } + proj.expressions = std::move(new_exprs); + + for (auto *above : above_projs) { + for (auto &expr : above->expressions) { + RewriteBindingRefs(*expr, proj.table_index.index, old_to_new); + } + } +} + +static void PruneColumnsAfterOrderByRemoval(LogicalOperator &child, LogicalGet &get, + const vector &projection_map) { + if (child.type == LogicalOperatorType::LOGICAL_GET) { + auto &column_ids = get.GetColumnIds(); + vector new_ids; + for (auto idx : projection_map) { + new_ids.push_back(column_ids[idx]); + } + get.SetColumnIds(std::move(new_ids)); + get.projection_ids.clear(); + return; + } + + if (child.type != LogicalOperatorType::LOGICAL_PROJECTION) { + return; + } + + vector proj_chain; + reference current = child; + while (current.get().type == LogicalOperatorType::LOGICAL_PROJECTION) { + proj_chain.push_back(¤t.get().Cast()); + if (current.get().children.empty()) { + break; + } + current = *current.get().children[0]; + } + + if (proj_chain.size() < 2) { + return; + } + + auto &top_proj = *proj_chain[0]; + vector above_projs; + above_projs.push_back(&top_proj); + + for (idx_t layer = 1; layer < proj_chain.size(); layer++) { + auto &prev_proj = *proj_chain[layer - 1]; + auto &curr_proj = *proj_chain[layer]; + + unordered_set needed; + for (auto &expr : prev_proj.expressions) { + CollectBindingRefs(*expr, curr_proj.table_index.index, needed); + } + + if (needed.size() < curr_proj.expressions.size()) { + PruneProjectionLayer(curr_proj, needed, above_projs); + } + above_projs.push_back(&curr_proj); + } + + auto &bottom_proj = *proj_chain.back(); + unordered_set get_referenced; + for (auto &expr : bottom_proj.expressions) { + CollectBindingRefs(*expr, get.table_index.index, get_referenced); + } + + auto &column_ids = get.GetColumnIds(); + if (get_referenced.size() < column_ids.size()) { + vector new_ids; + unordered_map get_old_to_new; + for (idx_t i = 0; i < column_ids.size(); i++) { + if (get_referenced.count(i)) { + get_old_to_new[i] = new_ids.size(); + new_ids.push_back(column_ids[i]); + } + } + get.SetColumnIds(std::move(new_ids)); + get.projection_ids.clear(); + + for (auto *proj : proj_chain) { + for (auto &expr : proj->expressions) { + RewriteBindingRefs(*expr, get.table_index.index, get_old_to_new); + } + } + } +} + +void OrderByAndLimitOptimizer::Optimize(const OrderByAndLimitOptimizer::Config &config, OptimizerExtensionInput &input, + unique_ptr &op) { + if (!config.enabled) { + return; + } + + if (op->type == LogicalOperatorType::LOGICAL_TOP_N) { + auto &topn = op->Cast(); + LogicalGet *get = nullptr; + dbconnector::BindData *bind_data = nullptr; + if (OptimizerUtil::FindExtensionGet(config.table_scan_name, *op->children[0], get, bind_data)) { + string order_clause = TryBuildOrderByClause(config, topn.orders, *op->children[0], *get); + if (!order_clause.empty()) { + auto &order_by_and_limit_bind_data = bind_data->GetOrderByAndLimitBindData(); + order_by_and_limit_bind_data.order_by_clause = order_clause; + order_by_and_limit_bind_data.limit_clause = " LIMIT " + to_string(topn.limit); + if (topn.offset > 0) { + order_by_and_limit_bind_data.limit_clause += " OFFSET " + to_string(topn.offset); + } + op = std::move(op->children[0]); + return; + } + } + for (auto &child : op->children) { + Optimize(config, input, child); + } + return; + } + if (op->type == LogicalOperatorType::LOGICAL_ORDER_BY) { + auto &order = op->Cast(); + LogicalGet *get = nullptr; + dbconnector::BindData *bind_data = nullptr; + if (OptimizerUtil::FindExtensionGet(config.table_scan_name, *op->children[0], get, bind_data)) { + string order_clause = TryBuildOrderByClause(config, order.orders, *op->children[0], *get); + if (!order_clause.empty()) { + auto &order_by_and_limit_bind_data = bind_data->GetOrderByAndLimitBindData(); + order_by_and_limit_bind_data.order_by_clause = order_clause; + if (!order.projection_map.empty()) { + vector indices; + indices.reserve(order.projection_map.size()); + for (auto proj_idx : order.projection_map) { + ColumnIndex col_idx = get->GetColumnIndex(proj_idx); + column_t table_col_idx = col_idx.GetPrimaryIndex(); + indices.emplace_back(table_col_idx); + } + PruneColumnsAfterOrderByRemoval(*op->children[0], *get, indices); + } + op = std::move(op->children[0]); + return; + } + } + for (auto &child : op->children) { + Optimize(config, input, child); + } + return; + } + if (op->type == LogicalOperatorType::LOGICAL_LIMIT) { + auto &limit = op->Cast(); + reference child = *op->children[0]; + while (child.get().type == LogicalOperatorType::LOGICAL_PROJECTION) { + child = *child.get().children[0]; + } + if (child.get().type != LogicalOperatorType::LOGICAL_GET) { + return; + } + auto &get = child.get().Cast(); + if (get.function.name != config.table_scan_name) { + return; + } + switch (limit.limit_val.Type()) { + case LimitNodeType::CONSTANT_VALUE: + case LimitNodeType::UNSET: + break; + default: + return; + } + switch (limit.offset_val.Type()) { + case LimitNodeType::CONSTANT_VALUE: + case LimitNodeType::UNSET: + break; + default: + return; + } + auto &bind_data = get.bind_data->Cast(); + auto &order_by_and_limit_bind_data = bind_data.GetOrderByAndLimitBindData(); + if (!order_by_and_limit_bind_data.limit_clause.empty()) { + return; + } + bool has_limit = (limit.limit_val.Type() != LimitNodeType::UNSET); + bool has_offset = (limit.offset_val.Type() != LimitNodeType::UNSET); + if (!has_limit && has_offset) { + return; + } + if (has_limit) { + order_by_and_limit_bind_data.limit_clause = " LIMIT " + to_string(limit.limit_val.GetConstantValue()); + } + if (has_offset) { + order_by_and_limit_bind_data.limit_clause += " OFFSET " + to_string(limit.offset_val.GetConstantValue()); + } + op = std::move(op->children[0]); + return; + } + for (auto &child : op->children) { + Optimize(config, input, child); + } +} + +} // namespace optimizer +} // namespace dbconnector diff --git a/src/query/query_writer.cpp b/src/query/query_writer.cpp new file mode 100644 index 0000000..55e1937 --- /dev/null +++ b/src/query/query_writer.cpp @@ -0,0 +1,66 @@ +#include "dbconnector/query/query_writer.hpp" + +#include + +namespace dbconnector { +namespace query { + +std::string QueryWriter::EscapeQuotes(const std::string &text, char quote) { + std::string result; + for (auto c : text) { + if (c == quote) { + result += "\\"; + result += quote; + } else if (c == '\\') { + result += "\\\\"; + } else { + result += c; + } + } + return result; +} + +std::string QueryWriter::WriteQuoted(const std::string &text, char quote) { + // 1. Escapes all occurences of 'quote' by escaping them with a backslash + // 2. Adds quotes around the string + return std::string(1, quote) + EscapeQuotes(text, quote) + std::string(1, quote); +} + +std::string QueryWriter::WriteIdentifier(const std::string &identifier, char identifier_quote) { + return WriteQuoted(identifier, identifier_quote); +} + +std::string QueryWriter::WriteLiteral(const std::string &identifier) { + return QueryWriter::WriteQuoted(identifier, '\''); +} + +std::string QueryWriter::EncodeBlob(const std::string &val) { + char const HEX_DIGITS[] = "0123456789ABCDEF"; + + std::string result = "x'"; + for (size_t i = 0; i < val.size(); i++) { + uint8_t byte_val = static_cast(val[i]); + result += HEX_DIGITS[(byte_val >> 4) & 0xf]; + result += HEX_DIGITS[byte_val & 0xf]; + } + result += "'"; + return result; +} + +std::string QueryWriter::WriteConstant(const duckdb::Value &val) { + if (val.type().IsNumeric() || val.type().id() == duckdb::LogicalTypeId::BOOLEAN) { + return val.ToSQLString(); + } + if (val.type().id() == duckdb::LogicalTypeId::BLOB) { + return EncodeBlob(duckdb::StringValue::Get(val)); + } + if (val.type().id() == duckdb::LogicalTypeId::TIMESTAMP_TZ) { + return val.DefaultCastAs(duckdb::LogicalType::TIMESTAMP) + .DefaultCastAs(duckdb::LogicalType::VARCHAR) + .ToSQLString(); + } + return val.DefaultCastAs(duckdb::LogicalType::VARCHAR).ToSQLString(); +} + +} // namespace query +} // namespace dbconnector diff --git a/src/table_scan/filter_pushdown.cpp b/src/table_scan/filter_pushdown.cpp new file mode 100644 index 0000000..0cd9ade --- /dev/null +++ b/src/table_scan/filter_pushdown.cpp @@ -0,0 +1,166 @@ +#include "dbconnector/table_scan/filter_pushdown.hpp" + +#include "duckdb/planner/expression/bound_comparison_expression.hpp" +#include "duckdb/planner/expression/bound_conjunction_expression.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" +#include "duckdb/planner/expression/bound_operator_expression.hpp" +#include "duckdb/planner/filter/table_filter_functions.hpp" +#include "duckdb/common/enum_util.hpp" +#include "duckdb/common/string_util.hpp" + +#include "dbconnector/query/query_writer.hpp" + +#include "dbconnector/table_scan/filter_util.hpp" +#include "dbconnector/table_scan/table_scan_exception.hpp" + +namespace dbconnector { +namespace table_scan { + +using namespace duckdb; + +std::string FilterPushdown::CreateExpression(const std::string &column_name, + const vector> &filters, const std::string &op) { + vector filter_entries; + for (auto &filter : filters) { + auto new_filter = TransformExpression(column_name, *filter); + if (new_filter.empty()) { + continue; + } + filter_entries.push_back(std::move(new_filter)); + } + if (filter_entries.empty()) { + return std::string(); + } + return "(" + StringUtil::Join(filter_entries, " " + op + " ") + ")"; +} + +std::string FilterPushdown::TransformComparison(ExpressionType type) { + switch (type) { + case ExpressionType::COMPARE_EQUAL: + return "="; + case ExpressionType::COMPARE_NOTEQUAL: + return "!="; + case ExpressionType::COMPARE_LESSTHAN: + return "<"; + case ExpressionType::COMPARE_GREATERTHAN: + return ">"; + case ExpressionType::COMPARE_LESSTHANOREQUALTO: + return "<="; + case ExpressionType::COMPARE_GREATERTHANOREQUALTO: + return ">="; + default: + throw TableScanException("Unsupported expression type: '" + EnumUtil::ToString(type) + "'"); + } +} + +static bool IsDirectReference(const Expression &expr) { + switch (expr.GetExpressionClass()) { + case ExpressionClass::BOUND_REF: + case ExpressionClass::BOUND_COLUMN_REF: + return true; + default: + return false; + } +} + +std::string FilterPushdown::TransformExpression(const std::string &column_name, const Expression &expr) { + if (BoundComparisonExpression::IsComparison(expr)) { + auto &comparison = expr.Cast(); + auto comparison_type = comparison.GetExpressionType(); + auto &left = BoundComparisonExpression::Left(comparison); + auto &right = BoundComparisonExpression::Right(comparison); + const Value *constant = nullptr; + if (IsDirectReference(left) && right.GetExpressionClass() == ExpressionClass::BOUND_CONSTANT) { + constant = &right.Cast().value; + } else if (left.GetExpressionClass() == ExpressionClass::BOUND_CONSTANT && IsDirectReference(right)) { + constant = &left.Cast().value; + comparison_type = FlipComparisonExpression(comparison_type); + } else { + return std::string(); + } + auto constant_string = query::QueryWriter::WriteConstant(*constant); + auto operator_string = TransformComparison(comparison_type); + return StringUtil::Format("%s %s %s", column_name, operator_string, constant_string); + } + + switch (expr.GetExpressionClass()) { + case ExpressionClass::BOUND_CONJUNCTION: { + auto &conjunction = expr.Cast(); + switch (conjunction.GetExpressionType()) { + case ExpressionType::CONJUNCTION_AND: + return CreateExpression(column_name, conjunction.children, "AND"); + case ExpressionType::CONJUNCTION_OR: + return CreateExpression(column_name, conjunction.children, "OR"); + default: + return std::string(); + } + } + case ExpressionClass::BOUND_OPERATOR: { + auto &op = expr.Cast(); + switch (op.GetExpressionType()) { + case ExpressionType::OPERATOR_IS_NULL: + if (op.children.size() == 1 && IsDirectReference(*op.children[0])) { + return column_name + " IS NULL"; + } + return std::string(); + case ExpressionType::OPERATOR_IS_NOT_NULL: + if (op.children.size() == 1 && IsDirectReference(*op.children[0])) { + return column_name + " IS NOT NULL"; + } + return std::string(); + case ExpressionType::COMPARE_IN: { + if (op.children.empty() || !IsDirectReference(*op.children[0])) { + return std::string(); + } + std::string in_list; + for (idx_t i = 1; i < op.children.size(); i++) { + if (op.children[i]->GetExpressionClass() != ExpressionClass::BOUND_CONSTANT) { + return std::string(); + } + if (!in_list.empty()) { + in_list += ", "; + } + in_list += query::QueryWriter::WriteConstant(op.children[i]->Cast().value); + } + return column_name + " IN (" + in_list + ")"; + } + default: + return std::string(); + } + } + case ExpressionClass::BOUND_FUNCTION: { + auto &func = expr.Cast(); + if (func.function.GetName() == OptionalFilterScalarFun::NAME && func.bind_info) { + auto &data = func.bind_info->Cast(); + return data.child_filter_expr ? TransformExpression(column_name, *data.child_filter_expr) : std::string(); + } + if (func.function.GetName() == SelectivityOptionalFilterScalarFun::NAME && func.bind_info) { + auto &data = func.bind_info->Cast(); + return data.child_filter_expr ? TransformExpression(column_name, *data.child_filter_expr) : std::string(); + } + if (func.function.GetName() == DynamicFilterScalarFun::NAME) { + return std::string(); + } + return std::string(); + } + default: + return std::string(); + } +} + +FilterPushdown::Config FilterPushdown::CreateConfig(char identifier_quote) { + Config res; + res.identifier_quote = identifier_quote; + return res; +} + +std::string FilterPushdown::TransformFilter(const FilterPushdown::Config &config, const std::string &column_name, + const TableFilter &filter) { + std::string column_name_quoted = query::QueryWriter::WriteIdentifier(column_name, config.identifier_quote); + auto &expr = FilterUtil::GetExpression(filter, "FilterPushdown::TransformFilter"); + return TransformExpression(column_name_quoted, expr); +} + +} // namespace table_scan +} // namespace dbconnector diff --git a/src/table_scan/filter_util.cpp b/src/table_scan/filter_util.cpp new file mode 100644 index 0000000..d6a1da2 --- /dev/null +++ b/src/table_scan/filter_util.cpp @@ -0,0 +1,26 @@ +#include "dbconnector/table_scan/filter_util.hpp" + +#include "duckdb/planner/filter/expression_filter.hpp" + +namespace dbconnector { +namespace table_scan { + +const duckdb::Expression &FilterUtil::GetExpression(const duckdb::TableFilter &filter, + const std::string &call_context) { + auto &expr_filter = duckdb::ExpressionFilter::GetExpressionFilter(filter, call_context.c_str()); + return *expr_filter.expr; +} + +bool FilterUtil::IsInternalFilter(const duckdb::TableFilter &filter) { + auto &expr = GetExpression(filter, "FilterPushdown::IsInternalFilter"); + return expr.GetExpressionClass() == duckdb::ExpressionClass::BOUND_FUNCTION && + expr.ToString().find("__internal_tablefilter_") == 0; +} + +std::string FilterUtil::ToString(const duckdb::TableFilter &filter) { + auto &expr = GetExpression(filter, "FilterPushdown::ToString"); + return expr.ToString(); +} + +} // namespace table_scan +} // namespace dbconnector diff --git a/test/extension/CMakeLists.txt b/test/extension/CMakeLists.txt new file mode 100644 index 0000000..3c491d7 --- /dev/null +++ b/test/extension/CMakeLists.txt @@ -0,0 +1,30 @@ +cmake_minimum_required(VERSION 3.5...3.29) + +project (test_database_connector_ext C CXX) + +if(MSVC) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR}) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR}) +endif() + +if(NOT DEFINED ENV{DUCKDB_SRC_PATH}) + message(FATAL_ERROR "'DUCKDB_SRC_PATH' environment variable must be defined") +endif() + +add_executable(${PROJECT_NAME} + ../../src/query/query_writer.cpp + ../test_common.cpp + test_query.cpp +) + +target_include_directories(${PROJECT_NAME} PRIVATE + $ENV{DUCKDB_SRC_PATH}/src/include + ../../src/include + ../catch + ../include +) + +target_link_libraries(${PROJECT_NAME} PRIVATE + $ENV{DUCKDB_SRC_PATH}/build/release/src/libduckdb.so +) diff --git a/test/extension/test_query.cpp b/test/extension/test_query.cpp new file mode 100644 index 0000000..e0d1a57 --- /dev/null +++ b/test/extension/test_query.cpp @@ -0,0 +1,10 @@ +#include "test_common.hpp" + +#include "dbconnector/query/query_writer.hpp" + +static const std::string group_name = "[query]"; + +TEST_CASE("Test query writer identifier", group_name) { + REQUIRE(dbconnector::query::QueryWriter::WriteIdentifier("foobar", '"') == "\"foobar\""); + REQUIRE(dbconnector::query::QueryWriter::WriteIdentifier("foo\"bar", '"') == "\"foo\\\"bar\""); +}