Optimizer and pushdown common code

.github/workflows/ConnectorTests.yml

@@ -9,7 +9,7 @@ jobs:
     name: Format Check
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - run: |
           python -m pip install --user clang_format==11.0.1
           make format-check
@@ -22,7 +22,7 @@ jobs:
       GEN: ninja
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Dependencies
         run: |
@@ -41,7 +41,7 @@ jobs:
     runs-on: windows-latest
     needs: linux-tests-amd64
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Build and Test
         shell: cmd
@@ -59,7 +59,7 @@ jobs:
     env:
       GEN: ninja
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Build and Test
         run: |
@@ -73,7 +73,7 @@ jobs:
       GEN: ninja
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Dependencies
         run: |
@@ -86,3 +86,65 @@ jobs:
       - name: Build and Test
         run: |
           make test_threads
+
+  linux-ext-amd64:
+    name: Linux Extension (amd64)
+    runs-on: ubuntu-latest
+    needs: linux-tests-amd64
+    env:
+      GEN: ninja
+      CC: 'ccache gcc'
+      CXX: 'ccache g++'
+      CCACHE_DIR: ${{ github.workspace }}/ccache
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Checkout Engine
+        uses: actions/checkout@v6
+        with:
+          path: 'duckdb'
+          ref: 'main'
+          repository: duckdb/duckdb
+
+      - name: Dependencies
+        run: |
+          echo "set man-db/auto-update false" | sudo debconf-communicate
+          sudo dpkg-reconfigure man-db
+          sudo apt-get update -y -q -o=Dpkg::Use-Pty=0
+          sudo apt-get install -y -q -o=Dpkg::Use-Pty=0 \
+            build-essential \
+            ccache \
+            cmake \
+            ninja-build
+
+      - name: Cache Key
+        id: cache_key
+        working-directory: duckdb
+        run: |
+          DUCKDB_VERSION=$(git rev-parse --short HEAD)
+          KEY="${{ runner.os }}-${{ runner.arch }}-$DUCKDB_VERSION"
+          echo "value=${KEY}" >> "${GITHUB_OUTPUT}"
+
+      - name: Restore Cache
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ github.workspace }}/ccache
+          key: ${{ steps.cache_key.outputs.value }}
+
+      - name: Build Engine
+        working-directory: duckdb
+        run: |
+          make release
+
+      - name: Save Cache
+        uses: actions/cache/save@v5
+        with:
+          path: ${{ github.workspace }}/ccache
+          key: ${{ steps.cache_key.outputs.value }}
+
+      - name: Build and Test Extension
+        env:
+          DUCKDB_SRC_PATH: ${{ github.workspace }}/duckdb
+        run: |
+          make test_ext

Makefile

@@ -30,8 +30,17 @@ build_threads:
 		cmake -DCMAKE_BUILD_TYPE=Debug $(GENERATOR) -DENABLE_THREAD_SANITIZER=ON ../../test && \
 		cmake --build . --config Debug
 
+build_ext:
+	mkdir -p build/debug
+	cd build/debug && \
+		cmake -DCMAKE_BUILD_TYPE=Debug $(GENERATOR) ../../test/extension && \
+		cmake --build . --config Debug
+
 test: build 
 	build/debug/test_database_connector
 
 test_threads: build_threads
 	build/debug/test_database_connector
+
+test_ext: build_ext
+	build/debug/test_database_connector_ext

src/include/dbconnector/bind_data.hpp

@@ -0,0 +1,17 @@
+#pragma once
+
+#include "duckdb/function/function.hpp"
+
+#include "dbconnector/optimizer/aggregate_bind_data.hpp"
+#include "dbconnector/optimizer/order_by_and_limit_bind_data.hpp"
+
+namespace dbconnector {
+
+class BindData : public duckdb::FunctionData {
+public:
+	virtual optimizer::AggregateBindData &GetAggregateBindData() = 0;
+
+	virtual optimizer::OrderByAndLimitBindData &GetOrderByAndLimitBindData() = 0;
+};
+
+} // namespace dbconnector

src/include/dbconnector/optimizer/aggregate_bind_data.hpp

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <string>
+
+namespace dbconnector {
+namespace optimizer {
+
+struct AggregateBindData {
+	std::string aggregate_select_list;
+	std::string group_by_clause;
+	std::string aggregate_where_clause;
+	bool has_aggregate_pushdown = false;
+};
+
+} // namespace optimizer
+} // namespace dbconnector

src/include/dbconnector/optimizer/aggregate_optimizer.hpp

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <string>
+
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/optimizer/optimizer_extension.hpp"
+
+namespace dbconnector {
+namespace optimizer {
+
+typedef bool (*should_push_aggregate_t)(duckdb::ClientContext &context, duckdb::LogicalAggregate &aggr);
+
+class AggregateOptimizer {
+public:
+	struct Config {
+		bool enabled = false;
+		char identifier_quote = '"';
+		std::string table_scan_name;
+		should_push_aggregate_t should_push_aggregate = nullptr;
+	};
+
+	static Config CreateConfig(duckdb::ClientContext &ctx, const std::string &enabled_option,
+
+	                           char identifier_quote, std::string table_scan_name,
+	                           should_push_aggregate_t should_push_aggregate = nullptr);
+
+	static void Optimize(const Config &config, duckdb::OptimizerExtensionInput &input,
+	                     duckdb::unique_ptr<duckdb::LogicalOperator> &op);
+};
+
+} // namespace optimizer
+} // namespace dbconnector

src/include/dbconnector/optimizer/optimizer_util.hpp

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <string>
+
+#include "duckdb/common/types.hpp"
+#include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/planner/operator/logical_get.hpp"
+#include "duckdb/planner/column_binding.hpp"
+
+#include "dbconnector/bind_data.hpp"
+
+namespace dbconnector {
+namespace optimizer {
+
+struct TracedBindingColumn {
+	std::string col_name;
+	duckdb::LogicalType col_type;
+
+	bool Found() {
+		return !col_name.empty();
+	}
+};
+
+class OptimizerUtil {
+public:
+	static bool FindExtensionGet(const std::string &table_scan_name, duckdb::LogicalOperator &start,
+	                             duckdb::LogicalGet *&get_out, dbconnector::BindData *&bind_out);
+
+	static TracedBindingColumn TraceBindingToColumn(duckdb::ColumnBinding binding, duckdb::LogicalOperator &child,
+	                                                duckdb::LogicalGet &get);
+};
+
+} // namespace optimizer
+} // namespace dbconnector

src/include/dbconnector/optimizer/order_by_and_limit_bind_data.hpp

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <string>
+
+namespace dbconnector {
+namespace optimizer {
+
+struct OrderByAndLimitBindData {
+	std::string limit_clause;
+	std::string order_by_clause;
+};
+
+} // namespace optimizer
+} // namespace dbconnector

src/include/dbconnector/optimizer/order_by_and_limit_optimizer.hpp

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <string>
+
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/optimizer/optimizer_extension.hpp"
+
+namespace dbconnector {
+namespace optimizer {
+
+class OrderByAndLimitOptimizer {
+public:
+	struct Config {
+		bool enabled = false;
+		char identifier_quote = '"';
+		std::string table_scan_name;
+	};
+
+	static Config CreateConfig(duckdb::ClientContext &ctx, const std::string &enabled_option, char identifier_quote,
+	                           std::string table_scan_name);
+
+	static void Optimize(const Config &config, duckdb::OptimizerExtensionInput &input,
+	                     duckdb::unique_ptr<duckdb::LogicalOperator> &op);
+};
+
+} // namespace optimizer
+} // namespace dbconnector

src/include/dbconnector/query/query_writer.hpp

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <string>
+
+#include "duckdb/common/types/value.hpp"
+
+namespace dbconnector {
+namespace query {
+
+class QueryWriter {
+public:
+	static std::string WriteIdentifier(const std::string &identifier, char identifier_quote);
+	static std::string WriteLiteral(const std::string &identifier);
+	static std::string WriteConstant(const duckdb::Value &val);
+
+private:
+	static std::string EncodeBlob(const std::string &val);
+	static std::string EscapeQuotes(const std::string &text, char quote);
+	static std::string WriteQuoted(const std::string &text, char quote);
+};
+
+} // namespace query
+} // namespace dbconnector

src/include/dbconnector/table_scan/filter_pushdown.hpp

@@ -0,0 +1,31 @@
+#pragma once
+
+#include <string>
+
+#include "duckdb/common/enums/expression_type.hpp"
+#include "duckdb/planner/table_filter.hpp"
+
+namespace dbconnector {
+namespace table_scan {
+
+class FilterPushdown {
+	struct Config {
+		char identifier_quote = '"';
+	};
+
+public:
+	static Config CreateConfig(char identifier_quote);
+
+	static std::string TransformFilter(const Config &config, const std::string &column_name,
+	                                   const duckdb::TableFilter &filter);
+
+private:
+	static std::string TransformExpression(const std::string &column_name, const duckdb::Expression &expr);
+	static std::string TransformComparison(duckdb::ExpressionType type);
+	static std::string CreateExpression(const std::string &column_name,
+	                                    const duckdb::vector<duckdb::unique_ptr<duckdb::Expression>> &filters,
+	                                    const std::string &op);
+};
+
+} // namespace table_scan
+} // namespace dbconnector

src/include/dbconnector/table_scan/filter_util.hpp

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <string>
+
+#include "duckdb/common/enums/expression_type.hpp"
+#include "duckdb/planner/table_filter.hpp"
+
+namespace dbconnector {
+namespace table_scan {
+
+class FilterUtil {
+public:
+	static const duckdb::Expression &GetExpression(const duckdb::TableFilter &filter, const std::string &call_context);
+
+	static bool IsInternalFilter(const duckdb::TableFilter &filter);
+
+	static std::string ToString(const duckdb::TableFilter &filter);
+};
+
+} // namespace table_scan
+} // namespace dbconnector

src/include/dbconnector/table_scan/table_scan_exception.hpp

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <stdexcept>
+
+namespace dbconnector {
+namespace table_scan {
+
+class TableScanException : public std::exception {
+protected:
+	std::string message;
+
+public:
+	TableScanException() = default;
+
+	TableScanException(const std::string &message) : message(message.data(), message.length()) {
+	}
+
+	virtual const char *what() const noexcept {
+		return message.c_str();
+	}
+};
+
+} // namespace table_scan
+} // namespace dbconnector