From 45be177749219d67e3a570a7a03fcb1c17630816 Mon Sep 17 00:00:00 2001
From: fuziontech <fuziontech@gmail.com>
Date: Tue, 18 Aug 2026 15:34:42 +0000
Subject: [PATCH 1/4] feat(tools): ensure-indexes subcommand for catalog
 maintenance

Codifies the DuckLake catalog secondary indexes as an idempotent routine.
The stock catalog schema has no indexes on the listing/compaction
predicates; on megaduck (677M stats rows, 499k snapshots) a per-query file
listing seq-scanned for minutes. The set is the union of the two ad-hoc
indexing efforts applied there (snapshot-visibility reads, compaction/metrics
scans, ClickHouse-reader per-file fetches), named to match what is already
live so adoption is a no-op.

Implementation notes:
- direct psycopg connection with autocommit: duckdb postgres_execute wraps
  statements in BEGIN, and CREATE INDEX CONCURRENTLY cannot run inside one
  (verified against megaduck);
- every statement is CONCURRENTLY + IF NOT EXISTS: no write blocking, safe
  on every maintenance pass;
- dry-run needs no catalog connection.

Co-authored-by: Shelley <shelley@exe.dev>
---
 pyproject.toml                          |   5 ++
 tests/unit/test_ducklake_maintenance.py |  65 ++++++++++++++
 tools/ducklake_maintenance.py           | 110 +++++++++++++++++++++++-
 uv.lock                                 |  69 +++++++++++++++
 4 files changed, 248 insertions(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index ff829fd..1ef5f38 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -40,6 +40,11 @@ dependencies = [
     "pyarrow>=18.0",
     "orjson>=3.10",
     "prometheus-client>=0.21",
+    # Raw Postgres driver for tools/ducklake_maintenance.py's ensure-indexes
+    # subcommand: CREATE INDEX CONCURRENTLY must run outside a transaction,
+    # and duckdb's postgres_execute always wraps in BEGIN. Binary wheel so the
+    # image needs no libpq toolchain.
+    "psycopg[binary]>=3.2",
     # Required by duckdb's Python conversion for TIMESTAMPTZ columns; the
     # stdlib zoneinfo module isn't accepted as a substitute as of 1.4.x.
     "pytz>=2024.1",
diff --git a/tests/unit/test_ducklake_maintenance.py b/tests/unit/test_ducklake_maintenance.py
index a3bf6fb..ff6a9e6 100644
--- a/tests/unit/test_ducklake_maintenance.py
+++ b/tests/unit/test_ducklake_maintenance.py
@@ -1681,3 +1681,68 @@ def test_raises_when_lock_held_by_another_session(self):
         conn.execute.return_value.fetchone.return_value = (False,)
         with pytest.raises(RuntimeError, match="advisory lock"):
             ducklake_maintenance._acquire_advisory_lock(conn)
+
+
+# ---------------------------------------------------------------------------
+# ensure-indexes
+# ---------------------------------------------------------------------------
+
+
+class TestCatalogIndexes:
+    def test_names_are_unique(self):
+        names = [name for name, _ in ducklake_maintenance.CATALOG_INDEXES]
+        assert len(names) == len(set(names))
+
+    def test_definitions_target_known_catalog_tables(self):
+        known = {
+            "ducklake_data_file",
+            "ducklake_delete_file",
+            "ducklake_file_column_stats",
+            "ducklake_file_partition_value",
+        }
+        for name, definition in ducklake_maintenance.CATALOG_INDEXES:
+            table = definition.split(" ")[0]
+            assert table in known, f"{name} targets unknown catalog table {table}"
+
+    def test_live_names_match_megaduck(self):
+        """The names double as the idempotency key (IF NOT EXISTS) — renaming
+        one would build a duplicate index on catalogs that already have it."""
+        names = {name for name, _ in ducklake_maintenance.CATALOG_INDEXES}
+        # Indexes applied by hand on megaduck 2026-08 (both efforts); the
+        # routine adopts them by name.
+        assert "ducklake_file_column_stats_table_file_idx" in names
+        assert "ducklake_data_file_snapshot_read_idx" in names
+
+
+class TestEnsureIndexes:
+    def _env(self, monkeypatch):
+        monkeypatch.setenv("DUCKLAKE_RDS_HOST", "catalog.internal")
+        monkeypatch.setenv("DUCKLAKE_RDS_PASSWORD", "secret")
+
+    def test_dry_run_connects_nowhere(self, monkeypatch, caplog):
+        self._env(monkeypatch)
+        with caplog.at_level(logging.INFO):
+            ducklake_maintenance.ensure_indexes(dry_run=True)
+        assert "[dry-run] CREATE INDEX CONCURRENTLY IF NOT EXISTS" in caplog.text
+        assert len(ducklake_maintenance.CATALOG_INDEXES) > 0
+
+    def test_real_run_executes_every_index(self, monkeypatch):
+        self._env(monkeypatch)
+        executed = []
+        cursor = MagicMock()
+        cursor.execute.side_effect = lambda stmt: executed.append(stmt)
+        conn = MagicMock()
+        conn.cursor.return_value.__enter__.return_value = cursor
+        conn.__enter__.return_value = conn
+        psycopg = MagicMock()
+        psycopg.connect.return_value = conn
+        monkeypatch.setitem(__import__("sys").modules, "psycopg", psycopg)
+
+        ducklake_maintenance.ensure_indexes(dry_run=False)
+
+        assert len(executed) == len(ducklake_maintenance.CATALOG_INDEXES)
+        for stmt in executed:
+            assert stmt.startswith("CREATE INDEX CONCURRENTLY IF NOT EXISTS")
+            assert " ON public.ducklake_" in stmt
+        psycopg.connect.assert_called_once()
+        assert psycopg.connect.call_args.kwargs.get("autocommit") is True
diff --git a/tools/ducklake_maintenance.py b/tools/ducklake_maintenance.py
index 579110d..1c20e0c 100644
--- a/tools/ducklake_maintenance.py
+++ b/tools/ducklake_maintenance.py
@@ -103,6 +103,56 @@ def _log_version() -> None:
 # directly) must use PG_CATALOG_SCHEMA, not METADATA_SCHEMA.
 PG_CATALOG_SCHEMA = "public"
 
+# ---------------------------------------------------------------------------
+# Catalog index recipes (ensure-indexes)
+# ---------------------------------------------------------------------------
+#
+# Why this exists: the DuckLake catalog schema ships without secondary
+# indexes, and its per-snapshot listing/compaction predicates seq-scan
+# otherwise. Learned the hard way on megaduck (2026-08): 677M
+# ducklake_file_column_stats rows meant a `WHERE table_id = ...` listing took
+# minutes per query, and the ClickHouse DuckLake reader's changed-snapshot
+# guard retried until it flapped. Every entry is CREATE INDEX CONCURRENTLY
+# IF NOT EXISTS: additive, idempotent, and never blocks catalog writers.
+#
+# CONCURRENTLY cannot run through duckdb's postgres_execute (the extension
+# wraps it in BEGIN ... which Postgres rejects), so ensure_indexes() talks to
+# the catalog over a direct psycopg connection with autocommit.
+#
+# Names match the indexes already live on megaduck so adoption there is a
+# no-op; the set is the union of the ClickHouse-reader and compaction/metrics
+# access patterns.
+CATALOG_INDEXES: tuple[tuple[str, str], ...] = (
+    # Snapshot-visibility reads (the hot path for every table listing):
+    #   WHERE table_id = ? AND begin_snapshot <= ? AND (end_snapshot IS NULL OR ? < end_snapshot)
+    ("ducklake_data_file_snapshot_read_idx", "ducklake_data_file (table_id, begin_snapshot, end_snapshot)"),
+    ("ducklake_delete_file_snapshot_read_idx", "ducklake_delete_file (table_id, begin_snapshot, end_snapshot)"),
+    # Live-file scans (end_snapshot IS NULL) ordered by size, used by tiered
+    # compaction and the metrics daemon:
+    (
+        "ducklake_data_file_compaction_idx",
+        "ducklake_data_file (table_id, end_snapshot, file_size_bytes) WHERE end_snapshot IS NULL",
+    ),
+    (
+        "ducklake_data_file_compaction_order_idx",
+        "ducklake_data_file (table_id, end_snapshot, file_size_bytes, begin_snapshot, row_id_start, data_file_id) "
+        "WHERE end_snapshot IS NULL",
+    ),
+    ("ducklake_delete_file_table_idx", "ducklake_delete_file (table_id, end_snapshot) WHERE end_snapshot IS NULL"),
+    (
+        "ducklake_delete_file_metrics_idx",
+        "ducklake_delete_file (table_id, end_snapshot, file_size_bytes) WHERE end_snapshot IS NULL",
+    ),
+    # Per-file lookups by data_file_id (ClickHouse file listing joins, orphan
+    # healing):
+    ("ducklake_file_column_stats_file_idx", "ducklake_file_column_stats (data_file_id)"),
+    ("ducklake_file_partition_value_file_idx", "ducklake_file_partition_value (data_file_id)"),
+    # Table-scoped per-file reads: `WHERE table_id = ? AND data_file_id IN
+    # (...)` (the ClickHouse reader's stats/partition fetches):
+    ("ducklake_file_column_stats_table_file_idx", "ducklake_file_column_stats (table_id, data_file_id)"),
+    ("ducklake_file_partition_value_table_file_idx", "ducklake_file_partition_value (table_id, data_file_id)"),
+)
+
 # Companion SQL file: header conventions plus runtime-loadable macros.
 MAINTENANCE_SQL_PATH = Path(__file__).resolve().parent / "ducklake_maintenance.sql"
 
@@ -637,6 +687,52 @@ def cleanup(conn: duckdb.DuckDBPyConnection, days: int, dry_run: bool) -> None:
         _log_cleanup_throughput("cleanup", len(result), elapsed, _scheduled_for_deletion_count(conn))
 
 
+def ensure_indexes(dry_run: bool) -> None:
+    """Create the catalog's secondary indexes (CATALOG_INDEXES) idempotently.
+
+    Runs over a direct psycopg connection with autocommit: CREATE INDEX
+    CONCURRENTLY cannot run inside a transaction, and duckdb's postgres_execute
+    wraps everything in BEGIN (verified against megaduck 2026-08). Each
+    statement is `IF NOT EXISTS`, so the routine is safe to run on every
+    maintenance pass and cheap when the catalog is already indexed.
+    """
+    import psycopg  # local import: only this subcommand needs a raw pg driver
+
+    log.info(
+        "%sensure-indexes: %d index(es) against catalog schema %s",
+        "[dry-run] " if dry_run else "",
+        len(CATALOG_INDEXES),
+        PG_CATALOG_SCHEMA,
+    )
+    if dry_run:
+        for name, definition in CATALOG_INDEXES:
+            log.info(
+                "[dry-run] CREATE INDEX CONCURRENTLY IF NOT EXISTS %s ON %s.%s",
+                name,
+                PG_CATALOG_SCHEMA,
+                definition,
+            )
+        return
+
+    dsn = (
+        f"host={_require('DUCKLAKE_RDS_HOST')} "
+        f"port={os.environ.get('DUCKLAKE_RDS_PORT', '5432')} "
+        f"dbname={os.environ.get('DUCKLAKE_RDS_DATABASE', 'ducklake')} "
+        f"user={os.environ.get('DUCKLAKE_RDS_USERNAME', 'ducklake')} "
+        f"password={_require('DUCKLAKE_RDS_PASSWORD')} "
+        "sslmode=require"
+    )
+
+    with psycopg.connect(dsn, autocommit=True) as conn:
+        with conn.cursor() as cur:
+            for name, definition in CATALOG_INDEXES:
+                statement = f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {name} ON {PG_CATALOG_SCHEMA}.{definition}"
+                log.info("ensure-indexes: %s", name)
+                t0 = time.monotonic()
+                cur.execute(statement)
+                log.info("ensure-indexes: %s done in %.1fs", name, time.monotonic() - t0)
+
+
 def cleanup_all(conn: duckdb.DuckDBPyConnection) -> None:
     """Delete all files scheduled for deletion regardless of age.
 
@@ -2422,6 +2518,13 @@ def build_parser() -> argparse.ArgumentParser:
     p.add_argument("--days", type=int, default=7)
     p.add_argument("--dry-run", action="store_true")
 
+    # ensure-indexes
+    p = sub.add_parser(
+        "ensure-indexes",
+        help="Create the catalog's secondary indexes (CATALOG_INDEXES) idempotently, CONCURRENTLY",
+    )
+    p.add_argument("--dry-run", action="store_true")
+
     # checkpoint
     sub.add_parser("checkpoint", help="CHECKPOINT (merge + expire + cleanup)")
 
@@ -2529,7 +2632,10 @@ def main(argv: list[str] | None = None) -> None:
     t0 = time.monotonic()
     status = "success"
     conn = None
-    conn = connect(debug=args.debug)
+    # ensure-indexes speaks raw psycopg to the catalog and needs neither the
+    # DuckDB session nor its S3 wiring — skip the duckdb connect for it.
+    if args.command != "ensure-indexes":
+        conn = connect(debug=args.debug)
     try:
         match args.command:
             case "expire":
@@ -2556,6 +2662,8 @@ def main(argv: list[str] | None = None) -> None:
                 fsck(conn, args.dry_run, args.max_iterations)
             case "orphans":
                 orphans(conn, args.dry_run)
+            case "ensure-indexes":
+                ensure_indexes(args.dry_run)
             case "maintain":
                 maintain(conn, args.days, args.dry_run)
             case "checkpoint":
diff --git a/uv.lock b/uv.lock
index 0ad522b..63c46ce 100644
--- a/uv.lock
+++ b/uv.lock
@@ -276,6 +276,7 @@ dependencies = [
     { name = "opentelemetry-sdk" },
     { name = "orjson" },
     { name = "prometheus-client" },
+    { name = "psycopg", extra = ["binary"] },
     { name = "pyarrow" },
     { name = "pyducklake" },
     { name = "pytz" },
@@ -304,6 +305,7 @@ requires-dist = [
     { name = "opentelemetry-sdk", specifier = ">=1.30" },
     { name = "orjson", specifier = ">=3.10" },
     { name = "prometheus-client", specifier = ">=0.21" },
+    { name = "psycopg", extras = ["binary"], specifier = ">=3.2" },
     { name = "pyarrow", specifier = ">=18.0" },
     { name = "pyducklake", specifier = ">=1.0.17" },
     { name = "pytz", specifier = ">=2024.1" },
@@ -523,6 +525,64 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" },
 ]
 
+[[package]]
+name = "psycopg"
+version = "3.3.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
+    { name = "tzdata", marker = "sys_platform == 'win32'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/db/2f/cb91e5502ec9de1de6f1b76cfbf69531932725361168bb06963620c77e2e/psycopg-3.3.4.tar.gz", hash = "sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc", size = 165799, upload-time = "2026-05-01T23:31:55.179Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5c/e0/7b3dee031daae7743609ce3c746565d4a3ed7c2c186479eb48e34e838c64/psycopg-3.3.4-py3-none-any.whl", hash = "sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a", size = 213001, upload-time = "2026-05-01T23:20:50.816Z" },
+]
+
+[package.optional-dependencies]
+binary = [
+    { name = "psycopg-binary", marker = "implementation_name != 'pypy'" },
+]
+
+[[package]]
+name = "psycopg-binary"
+version = "3.3.4"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/95/7d/03818e13ba7f36de93573c93ee3482006d3dfa8b0f8d28df511bad0a1a92/psycopg_binary-3.3.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5ab28a2a7649df3b72e6b674b4c190e448e8e77cf496a65bd846472048de2089", size = 4591122, upload-time = "2026-05-01T23:27:56.162Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/b9/11b341edf8d54e2694726b273fe9652b254d989f4f63e3ac6816ad6b55f4/psycopg_binary-3.3.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6402a9d8146cf4b3974ded3fd28a971e83dc6a0333eb7822524a3aa20b546578", size = 4669943, upload-time = "2026-05-01T23:28:04.522Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/18/4665bacd65e7865b4372fcd8abb8b9186ada4b0025f8c2ca691b364a556c/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:580ae30a5f95ccd90008ec697d3ed6a4a2047a516407ad904283fa42086936e9", size = 5469697, upload-time = "2026-05-01T23:28:11.337Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/b1/b83136c6e510593d9b0c759ba5384337bc4ad82d19fda675adc4b2703c84/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e7510c37550f91a187e3660a8cc50d4b760f8c3b8b2f89ebc5698cd2c7f2c85d", size = 5152995, upload-time = "2026-05-01T23:28:20.529Z" },
+    { url = "https://files.pythonhosted.org/packages/67/8d/a9821e2a648afe6091989929982a3b0f00b2631a859cb81379728f08fb75/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77df19583501ea288eaf15ac0fe7ad01e6d8091a91d5c41df5c718f307d8e31b", size = 6738180, upload-time = "2026-05-01T23:28:30.654Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/58/2e349e8d23905dc2317b80ac65f48fb6f821a4777a4e994a60da91c4850f/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:018fbed325936da502feb546642c982dcc4b9ffdea32dfef78dbf3b7f7ad4070", size = 4978828, upload-time = "2026-05-01T23:28:37.277Z" },
+    { url = "https://files.pythonhosted.org/packages/45/48/57b00d03b4721878326122a1f1e6b0a90b85bcaec56b5b2f8ea6cfa45235/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:17a21953a9e5ff3a16dab692625a3676e2f101db5e40072f39dbee2250194d68", size = 4509757, upload-time = "2026-05-01T23:28:43.078Z" },
+    { url = "https://files.pythonhosted.org/packages/25/37/33b47d8c007df69aec500df5889767c4d313748e8e9e27a2fef8a6dabcee/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:eb05ee1c2b817d27c537333224c9e83c7afb86fe7296ba970990068baf819b16", size = 4190546, upload-time = "2026-05-01T23:28:50.016Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/c6/32b0835dbc2122617902b649d76a91c1e75406e76bf3d595b0c3bb5ffad6/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:773d573e11f437ce0bdb95b7c18dc58390494f96d43f8b45b9760436114f7652", size = 3926197, upload-time = "2026-05-01T23:28:55.55Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/68/d190ef0c0c5b16ded07831dabc8ddd412f4cdab07ec6e30ed38d9bda0e1f/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e55ccbdfae79a2ed9c6369c3008a3025817ff9d7e27b32a2d84e2a4267e66e", size = 4236627, upload-time = "2026-05-01T23:29:05.336Z" },
+    { url = "https://files.pythonhosted.org/packages/25/8f/81dcbc2e8454b74d14881275ea45f00791052dac531a9fa8be1730d1685b/psycopg_binary-3.3.4-cp312-cp312-win_amd64.whl", hash = "sha256:494ca54901be8cf9eb7e02c25b731f2317c378efa44f43e8f9bd0e1184ae7be4", size = 3560782, upload-time = "2026-05-01T23:29:11.967Z" },
+    { url = "https://files.pythonhosted.org/packages/09/43/13e9c406fbbf354580476e248a16b64802a376873ebe6339e30bb655572d/psycopg_binary-3.3.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7", size = 4590377, upload-time = "2026-05-01T23:29:18.782Z" },
+    { url = "https://files.pythonhosted.org/packages/22/be/2923cd7c3683e7afdecf4f10796a18de02f5c5ddc0969aa2ad0a8cdd3bbd/psycopg_binary-3.3.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:75a9067e236f9b9ae3535b66fe99bddb33d39c0de10112e49b9ab11eee53dc31", size = 4669023, upload-time = "2026-05-01T23:29:25.884Z" },
+    { url = "https://files.pythonhosted.org/packages/96/a0/2c913d6fe13d6a8bd13597d36739bf47af063ad9399e402cfecab16f3c1e/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b56b603ebcea8aa10b46228b8410ba7f13e7c2ee54389d4d9be0927fd8ce2a70", size = 5467423, upload-time = "2026-05-01T23:29:33.416Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/38/205d10bc1ad0df4a21c5c51659126bd3ea0ef98fcad1e852f78c249bb9c3/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c677c4ad433cb7150c8cd304a0769ae3bcfbe5ea0676eb53faa7b1443b16d0d3", size = 5151137, upload-time = "2026-05-01T23:29:42.013Z" },
+    { url = "https://files.pythonhosted.org/packages/36/fc/f0381ddcd45eff3bb70dbca6823a996048d7f507b2ec3fc92c6fabc0fe87/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26df2717e59c0473e4465a97dfb1b7afebaa479277870fd5784d1436470db47c", size = 6736671, upload-time = "2026-05-01T23:29:51.626Z" },
+    { url = "https://files.pythonhosted.org/packages/95/40/fa545ae152c24327651e5624e4902121e808270be36c10b12e9939be09bc/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dc1f79fd16bb1f3f4421417a514607539f17804d95c7ed617265369d1981cae", size = 4979601, upload-time = "2026-05-01T23:29:56.961Z" },
+    { url = "https://files.pythonhosted.org/packages/86/e4/2f8a47ee97f90cd2b933d0463081d35631ff419de2b8c984a5f369857de0/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:136f199a407b5348b9b857c504aff60c77622a28482e7195839ce1b51238c4cc", size = 4510513, upload-time = "2026-05-01T23:30:07.243Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/0e/94e842ff4a7f98ed162580ca2e8b8864b28c1e0350f2443f8ee47f821167/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b6f5a29e9c775b9f12a1a717aa7a2c80f9e1db6f27ba44a5b59c80ac61d2ffcf", size = 4187243, upload-time = "2026-05-01T23:30:15.352Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/83/fc6c174b672e29b7de996ea77b6cbddf46c891751c3355f6974292baa6b4/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:ee17a2cf4943cde261adfad1bbc5bf38d6b3776d7afff74c7cabcbeaeb08c260", size = 3927347, upload-time = "2026-05-01T23:30:21.186Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/65/768364d4a97a15b1a7f47ba52688c1686f22941d8332a8398cefc468e25f/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c4ab71be17bdca30cb34c34c4e1496e2f5d6f20c199c12bad226070b22ef9bf", size = 4236393, upload-time = "2026-05-01T23:30:26.211Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/3b/218efbc9e645becd80cdf651acda05f85cfe546b7a9c0458c7cbc8fe1f74/psycopg_binary-3.3.4-cp313-cp313-win_amd64.whl", hash = "sha256:dbfdb9b6cc79f31104a7b162a2b921b765fcc62af6c00540a167a8de47e4ed38", size = 3564592, upload-time = "2026-05-01T23:30:31.764Z" },
+    { url = "https://files.pythonhosted.org/packages/48/a6/828c9185701dab71b234c2a76c38a08b098ebfec5020716b4e93807492b5/psycopg_binary-3.3.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:28b7398fdd19db3232c884fb24550bdfe951221f510e195e233299e4c9b78f97", size = 4607292, upload-time = "2026-05-01T23:30:38.962Z" },
+    { url = "https://files.pythonhosted.org/packages/92/58/5b40dbc9d839045c9dae956960e4fb6d20bcabe6c59a2aa34fc3a371913f/psycopg_binary-3.3.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1fbaa292a3c8bb61b45df1ad3da1908ccee7cb889db9425e3557d9e34e2a4829", size = 4687023, upload-time = "2026-05-01T23:30:47.227Z" },
+    { url = "https://files.pythonhosted.org/packages/85/a9/793f0ac107a9003b48441d0d1f9f616d96e0f37458dd8dc12528ceff55fb/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94596f9e7633ee3f6440711d43bb70aa31cc0a46a900ab8b4201a366ace5c9e7", size = 5486985, upload-time = "2026-05-01T23:30:55.517Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/26/42e8533497e2592334f68ec529cf5f840f7fa4e99575a4bb61aa184dbfbf/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8c0056529e68dbe9184cd4019a1f3d8f3a4ead2f6fc7a5afcf27d3314edd1277", size = 5168745, upload-time = "2026-05-01T23:31:01.904Z" },
+    { url = "https://files.pythonhosted.org/packages/15/af/b7151776cc08d5935d45c833ec818a9beb417cf7c08239af1aafbdae78ee/psycopg_binary-3.3.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c09aad7051326e7603c14e50636db9c01f78272dc54b3accff03d46370461e6", size = 6761486, upload-time = "2026-05-01T23:31:14.511Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/ed/c92533b9124712d592cbf1cd6c76da933a2e0acea81dfe1fbe7e735f0cff/psycopg_binary-3.3.4-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:514404ed543efd620c85602b747df2a23cf1241b4067199e1a66f2d2757aaa41", size = 4997427, upload-time = "2026-05-01T23:31:20.901Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/23/ccadfd0de416aa188356daa199453af24087b042e296088706d190ae0295/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:46893c26858be12cc49ca4226ed6a60b4bfccadd946b3bebb783a60b38788228", size = 4533549, upload-time = "2026-05-01T23:31:26.204Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/a0/c8f43cee36386f7bc891ab41a9d31ea07cf9826038e732da79f26b1e5f34/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:df1d567fc430f6df15c9fcf67d87685fc49bdb325adc0db5af1adfb2f44eb5c9", size = 4210256, upload-time = "2026-05-01T23:31:33.884Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/2c/c1547871be3790676e8868b38655496422f94f0978dfb66b74bdba2f1676/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:6b9016b1714da4dd5ecaaa75b82098aa5a0b87854ce9b092e21c27c4ae23e014", size = 3946204, upload-time = "2026-05-01T23:31:39.626Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/b1/f6670f00fa7ea601584623f6c11602ab92117d83eaff885e0210f6de7418/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:47c656a8a7ba6eb0cff1801a4caaa9c8bdc12d03080e273aff1c8ac39971a77e", size = 4255811, upload-time = "2026-05-01T23:31:44.986Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" },
+]
+
 [[package]]
 name = "pyarrow"
 version = "23.0.1"
@@ -783,6 +843,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" },
 ]
 
+[[package]]
+name = "tzdata"
+version = "2026.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" },
+]
+
 [[package]]
 name = "urllib3"
 version = "2.6.3"

From 1d5643f68292dbd001d42fb7bb993bca1588ed8a Mon Sep 17 00:00:00 2001
From: fuziontech <fuziontech@gmail.com>
Date: Tue, 18 Aug 2026 16:55:47 +0000
Subject: [PATCH 2/4] Revert "feat(tools): ensure-indexes subcommand for
 catalog maintenance"

This reverts commit 45be177749219d67e3a570a7a03fcb1c17630816.
---
 pyproject.toml                          |   5 --
 tests/unit/test_ducklake_maintenance.py |  65 --------------
 tools/ducklake_maintenance.py           | 110 +-----------------------
 uv.lock                                 |  69 ---------------
 4 files changed, 1 insertion(+), 248 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 1ef5f38..ff829fd 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -40,11 +40,6 @@ dependencies = [
     "pyarrow>=18.0",
     "orjson>=3.10",
     "prometheus-client>=0.21",
-    # Raw Postgres driver for tools/ducklake_maintenance.py's ensure-indexes
-    # subcommand: CREATE INDEX CONCURRENTLY must run outside a transaction,
-    # and duckdb's postgres_execute always wraps in BEGIN. Binary wheel so the
-    # image needs no libpq toolchain.
-    "psycopg[binary]>=3.2",
     # Required by duckdb's Python conversion for TIMESTAMPTZ columns; the
     # stdlib zoneinfo module isn't accepted as a substitute as of 1.4.x.
     "pytz>=2024.1",
diff --git a/tests/unit/test_ducklake_maintenance.py b/tests/unit/test_ducklake_maintenance.py
index ff6a9e6..a3bf6fb 100644
--- a/tests/unit/test_ducklake_maintenance.py
+++ b/tests/unit/test_ducklake_maintenance.py
@@ -1681,68 +1681,3 @@ def test_raises_when_lock_held_by_another_session(self):
         conn.execute.return_value.fetchone.return_value = (False,)
         with pytest.raises(RuntimeError, match="advisory lock"):
             ducklake_maintenance._acquire_advisory_lock(conn)
-
-
-# ---------------------------------------------------------------------------
-# ensure-indexes
-# ---------------------------------------------------------------------------
-
-
-class TestCatalogIndexes:
-    def test_names_are_unique(self):
-        names = [name for name, _ in ducklake_maintenance.CATALOG_INDEXES]
-        assert len(names) == len(set(names))
-
-    def test_definitions_target_known_catalog_tables(self):
-        known = {
-            "ducklake_data_file",
-            "ducklake_delete_file",
-            "ducklake_file_column_stats",
-            "ducklake_file_partition_value",
-        }
-        for name, definition in ducklake_maintenance.CATALOG_INDEXES:
-            table = definition.split(" ")[0]
-            assert table in known, f"{name} targets unknown catalog table {table}"
-
-    def test_live_names_match_megaduck(self):
-        """The names double as the idempotency key (IF NOT EXISTS) — renaming
-        one would build a duplicate index on catalogs that already have it."""
-        names = {name for name, _ in ducklake_maintenance.CATALOG_INDEXES}
-        # Indexes applied by hand on megaduck 2026-08 (both efforts); the
-        # routine adopts them by name.
-        assert "ducklake_file_column_stats_table_file_idx" in names
-        assert "ducklake_data_file_snapshot_read_idx" in names
-
-
-class TestEnsureIndexes:
-    def _env(self, monkeypatch):
-        monkeypatch.setenv("DUCKLAKE_RDS_HOST", "catalog.internal")
-        monkeypatch.setenv("DUCKLAKE_RDS_PASSWORD", "secret")
-
-    def test_dry_run_connects_nowhere(self, monkeypatch, caplog):
-        self._env(monkeypatch)
-        with caplog.at_level(logging.INFO):
-            ducklake_maintenance.ensure_indexes(dry_run=True)
-        assert "[dry-run] CREATE INDEX CONCURRENTLY IF NOT EXISTS" in caplog.text
-        assert len(ducklake_maintenance.CATALOG_INDEXES) > 0
-
-    def test_real_run_executes_every_index(self, monkeypatch):
-        self._env(monkeypatch)
-        executed = []
-        cursor = MagicMock()
-        cursor.execute.side_effect = lambda stmt: executed.append(stmt)
-        conn = MagicMock()
-        conn.cursor.return_value.__enter__.return_value = cursor
-        conn.__enter__.return_value = conn
-        psycopg = MagicMock()
-        psycopg.connect.return_value = conn
-        monkeypatch.setitem(__import__("sys").modules, "psycopg", psycopg)
-
-        ducklake_maintenance.ensure_indexes(dry_run=False)
-
-        assert len(executed) == len(ducklake_maintenance.CATALOG_INDEXES)
-        for stmt in executed:
-            assert stmt.startswith("CREATE INDEX CONCURRENTLY IF NOT EXISTS")
-            assert " ON public.ducklake_" in stmt
-        psycopg.connect.assert_called_once()
-        assert psycopg.connect.call_args.kwargs.get("autocommit") is True
diff --git a/tools/ducklake_maintenance.py b/tools/ducklake_maintenance.py
index 1c20e0c..579110d 100644
--- a/tools/ducklake_maintenance.py
+++ b/tools/ducklake_maintenance.py
@@ -103,56 +103,6 @@ def _log_version() -> None:
 # directly) must use PG_CATALOG_SCHEMA, not METADATA_SCHEMA.
 PG_CATALOG_SCHEMA = "public"
 
-# ---------------------------------------------------------------------------
-# Catalog index recipes (ensure-indexes)
-# ---------------------------------------------------------------------------
-#
-# Why this exists: the DuckLake catalog schema ships without secondary
-# indexes, and its per-snapshot listing/compaction predicates seq-scan
-# otherwise. Learned the hard way on megaduck (2026-08): 677M
-# ducklake_file_column_stats rows meant a `WHERE table_id = ...` listing took
-# minutes per query, and the ClickHouse DuckLake reader's changed-snapshot
-# guard retried until it flapped. Every entry is CREATE INDEX CONCURRENTLY
-# IF NOT EXISTS: additive, idempotent, and never blocks catalog writers.
-#
-# CONCURRENTLY cannot run through duckdb's postgres_execute (the extension
-# wraps it in BEGIN ... which Postgres rejects), so ensure_indexes() talks to
-# the catalog over a direct psycopg connection with autocommit.
-#
-# Names match the indexes already live on megaduck so adoption there is a
-# no-op; the set is the union of the ClickHouse-reader and compaction/metrics
-# access patterns.
-CATALOG_INDEXES: tuple[tuple[str, str], ...] = (
-    # Snapshot-visibility reads (the hot path for every table listing):
-    #   WHERE table_id = ? AND begin_snapshot <= ? AND (end_snapshot IS NULL OR ? < end_snapshot)
-    ("ducklake_data_file_snapshot_read_idx", "ducklake_data_file (table_id, begin_snapshot, end_snapshot)"),
-    ("ducklake_delete_file_snapshot_read_idx", "ducklake_delete_file (table_id, begin_snapshot, end_snapshot)"),
-    # Live-file scans (end_snapshot IS NULL) ordered by size, used by tiered
-    # compaction and the metrics daemon:
-    (
-        "ducklake_data_file_compaction_idx",
-        "ducklake_data_file (table_id, end_snapshot, file_size_bytes) WHERE end_snapshot IS NULL",
-    ),
-    (
-        "ducklake_data_file_compaction_order_idx",
-        "ducklake_data_file (table_id, end_snapshot, file_size_bytes, begin_snapshot, row_id_start, data_file_id) "
-        "WHERE end_snapshot IS NULL",
-    ),
-    ("ducklake_delete_file_table_idx", "ducklake_delete_file (table_id, end_snapshot) WHERE end_snapshot IS NULL"),
-    (
-        "ducklake_delete_file_metrics_idx",
-        "ducklake_delete_file (table_id, end_snapshot, file_size_bytes) WHERE end_snapshot IS NULL",
-    ),
-    # Per-file lookups by data_file_id (ClickHouse file listing joins, orphan
-    # healing):
-    ("ducklake_file_column_stats_file_idx", "ducklake_file_column_stats (data_file_id)"),
-    ("ducklake_file_partition_value_file_idx", "ducklake_file_partition_value (data_file_id)"),
-    # Table-scoped per-file reads: `WHERE table_id = ? AND data_file_id IN
-    # (...)` (the ClickHouse reader's stats/partition fetches):
-    ("ducklake_file_column_stats_table_file_idx", "ducklake_file_column_stats (table_id, data_file_id)"),
-    ("ducklake_file_partition_value_table_file_idx", "ducklake_file_partition_value (table_id, data_file_id)"),
-)
-
 # Companion SQL file: header conventions plus runtime-loadable macros.
 MAINTENANCE_SQL_PATH = Path(__file__).resolve().parent / "ducklake_maintenance.sql"
 
@@ -687,52 +637,6 @@ def cleanup(conn: duckdb.DuckDBPyConnection, days: int, dry_run: bool) -> None:
         _log_cleanup_throughput("cleanup", len(result), elapsed, _scheduled_for_deletion_count(conn))
 
 
-def ensure_indexes(dry_run: bool) -> None:
-    """Create the catalog's secondary indexes (CATALOG_INDEXES) idempotently.
-
-    Runs over a direct psycopg connection with autocommit: CREATE INDEX
-    CONCURRENTLY cannot run inside a transaction, and duckdb's postgres_execute
-    wraps everything in BEGIN (verified against megaduck 2026-08). Each
-    statement is `IF NOT EXISTS`, so the routine is safe to run on every
-    maintenance pass and cheap when the catalog is already indexed.
-    """
-    import psycopg  # local import: only this subcommand needs a raw pg driver
-
-    log.info(
-        "%sensure-indexes: %d index(es) against catalog schema %s",
-        "[dry-run] " if dry_run else "",
-        len(CATALOG_INDEXES),
-        PG_CATALOG_SCHEMA,
-    )
-    if dry_run:
-        for name, definition in CATALOG_INDEXES:
-            log.info(
-                "[dry-run] CREATE INDEX CONCURRENTLY IF NOT EXISTS %s ON %s.%s",
-                name,
-                PG_CATALOG_SCHEMA,
-                definition,
-            )
-        return
-
-    dsn = (
-        f"host={_require('DUCKLAKE_RDS_HOST')} "
-        f"port={os.environ.get('DUCKLAKE_RDS_PORT', '5432')} "
-        f"dbname={os.environ.get('DUCKLAKE_RDS_DATABASE', 'ducklake')} "
-        f"user={os.environ.get('DUCKLAKE_RDS_USERNAME', 'ducklake')} "
-        f"password={_require('DUCKLAKE_RDS_PASSWORD')} "
-        "sslmode=require"
-    )
-
-    with psycopg.connect(dsn, autocommit=True) as conn:
-        with conn.cursor() as cur:
-            for name, definition in CATALOG_INDEXES:
-                statement = f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {name} ON {PG_CATALOG_SCHEMA}.{definition}"
-                log.info("ensure-indexes: %s", name)
-                t0 = time.monotonic()
-                cur.execute(statement)
-                log.info("ensure-indexes: %s done in %.1fs", name, time.monotonic() - t0)
-
-
 def cleanup_all(conn: duckdb.DuckDBPyConnection) -> None:
     """Delete all files scheduled for deletion regardless of age.
 
@@ -2518,13 +2422,6 @@ def build_parser() -> argparse.ArgumentParser:
     p.add_argument("--days", type=int, default=7)
     p.add_argument("--dry-run", action="store_true")
 
-    # ensure-indexes
-    p = sub.add_parser(
-        "ensure-indexes",
-        help="Create the catalog's secondary indexes (CATALOG_INDEXES) idempotently, CONCURRENTLY",
-    )
-    p.add_argument("--dry-run", action="store_true")
-
     # checkpoint
     sub.add_parser("checkpoint", help="CHECKPOINT (merge + expire + cleanup)")
 
@@ -2632,10 +2529,7 @@ def main(argv: list[str] | None = None) -> None:
     t0 = time.monotonic()
     status = "success"
     conn = None
-    # ensure-indexes speaks raw psycopg to the catalog and needs neither the
-    # DuckDB session nor its S3 wiring — skip the duckdb connect for it.
-    if args.command != "ensure-indexes":
-        conn = connect(debug=args.debug)
+    conn = connect(debug=args.debug)
     try:
         match args.command:
             case "expire":
@@ -2662,8 +2556,6 @@ def main(argv: list[str] | None = None) -> None:
                 fsck(conn, args.dry_run, args.max_iterations)
             case "orphans":
                 orphans(conn, args.dry_run)
-            case "ensure-indexes":
-                ensure_indexes(args.dry_run)
             case "maintain":
                 maintain(conn, args.days, args.dry_run)
             case "checkpoint":
diff --git a/uv.lock b/uv.lock
index 63c46ce..0ad522b 100644
--- a/uv.lock
+++ b/uv.lock
@@ -276,7 +276,6 @@ dependencies = [
     { name = "opentelemetry-sdk" },
     { name = "orjson" },
     { name = "prometheus-client" },
-    { name = "psycopg", extra = ["binary"] },
     { name = "pyarrow" },
     { name = "pyducklake" },
     { name = "pytz" },
@@ -305,7 +304,6 @@ requires-dist = [
     { name = "opentelemetry-sdk", specifier = ">=1.30" },
     { name = "orjson", specifier = ">=3.10" },
     { name = "prometheus-client", specifier = ">=0.21" },
-    { name = "psycopg", extras = ["binary"], specifier = ">=3.2" },
     { name = "pyarrow", specifier = ">=18.0" },
     { name = "pyducklake", specifier = ">=1.0.17" },
     { name = "pytz", specifier = ">=2024.1" },
@@ -525,64 +523,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" },
 ]
 
-[[package]]
-name = "psycopg"
-version = "3.3.4"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
-    { name = "tzdata", marker = "sys_platform == 'win32'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/db/2f/cb91e5502ec9de1de6f1b76cfbf69531932725361168bb06963620c77e2e/psycopg-3.3.4.tar.gz", hash = "sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc", size = 165799, upload-time = "2026-05-01T23:31:55.179Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/5c/e0/7b3dee031daae7743609ce3c746565d4a3ed7c2c186479eb48e34e838c64/psycopg-3.3.4-py3-none-any.whl", hash = "sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a", size = 213001, upload-time = "2026-05-01T23:20:50.816Z" },
-]
-
-[package.optional-dependencies]
-binary = [
-    { name = "psycopg-binary", marker = "implementation_name != 'pypy'" },
-]
-
-[[package]]
-name = "psycopg-binary"
-version = "3.3.4"
-source = { registry = "https://pypi.org/simple" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/95/7d/03818e13ba7f36de93573c93ee3482006d3dfa8b0f8d28df511bad0a1a92/psycopg_binary-3.3.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5ab28a2a7649df3b72e6b674b4c190e448e8e77cf496a65bd846472048de2089", size = 4591122, upload-time = "2026-05-01T23:27:56.162Z" },
-    { url = "https://files.pythonhosted.org/packages/a5/b9/11b341edf8d54e2694726b273fe9652b254d989f4f63e3ac6816ad6b55f4/psycopg_binary-3.3.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6402a9d8146cf4b3974ded3fd28a971e83dc6a0333eb7822524a3aa20b546578", size = 4669943, upload-time = "2026-05-01T23:28:04.522Z" },
-    { url = "https://files.pythonhosted.org/packages/8b/18/4665bacd65e7865b4372fcd8abb8b9186ada4b0025f8c2ca691b364a556c/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:580ae30a5f95ccd90008ec697d3ed6a4a2047a516407ad904283fa42086936e9", size = 5469697, upload-time = "2026-05-01T23:28:11.337Z" },
-    { url = "https://files.pythonhosted.org/packages/7c/b1/b83136c6e510593d9b0c759ba5384337bc4ad82d19fda675adc4b2703c84/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e7510c37550f91a187e3660a8cc50d4b760f8c3b8b2f89ebc5698cd2c7f2c85d", size = 5152995, upload-time = "2026-05-01T23:28:20.529Z" },
-    { url = "https://files.pythonhosted.org/packages/67/8d/a9821e2a648afe6091989929982a3b0f00b2631a859cb81379728f08fb75/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77df19583501ea288eaf15ac0fe7ad01e6d8091a91d5c41df5c718f307d8e31b", size = 6738180, upload-time = "2026-05-01T23:28:30.654Z" },
-    { url = "https://files.pythonhosted.org/packages/7e/58/2e349e8d23905dc2317b80ac65f48fb6f821a4777a4e994a60da91c4850f/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:018fbed325936da502feb546642c982dcc4b9ffdea32dfef78dbf3b7f7ad4070", size = 4978828, upload-time = "2026-05-01T23:28:37.277Z" },
-    { url = "https://files.pythonhosted.org/packages/45/48/57b00d03b4721878326122a1f1e6b0a90b85bcaec56b5b2f8ea6cfa45235/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:17a21953a9e5ff3a16dab692625a3676e2f101db5e40072f39dbee2250194d68", size = 4509757, upload-time = "2026-05-01T23:28:43.078Z" },
-    { url = "https://files.pythonhosted.org/packages/25/37/33b47d8c007df69aec500df5889767c4d313748e8e9e27a2fef8a6dabcee/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:eb05ee1c2b817d27c537333224c9e83c7afb86fe7296ba970990068baf819b16", size = 4190546, upload-time = "2026-05-01T23:28:50.016Z" },
-    { url = "https://files.pythonhosted.org/packages/ca/c6/32b0835dbc2122617902b649d76a91c1e75406e76bf3d595b0c3bb5ffad6/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:773d573e11f437ce0bdb95b7c18dc58390494f96d43f8b45b9760436114f7652", size = 3926197, upload-time = "2026-05-01T23:28:55.55Z" },
-    { url = "https://files.pythonhosted.org/packages/cd/68/d190ef0c0c5b16ded07831dabc8ddd412f4cdab07ec6e30ed38d9bda0e1f/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e55ccbdfae79a2ed9c6369c3008a3025817ff9d7e27b32a2d84e2a4267e66e", size = 4236627, upload-time = "2026-05-01T23:29:05.336Z" },
-    { url = "https://files.pythonhosted.org/packages/25/8f/81dcbc2e8454b74d14881275ea45f00791052dac531a9fa8be1730d1685b/psycopg_binary-3.3.4-cp312-cp312-win_amd64.whl", hash = "sha256:494ca54901be8cf9eb7e02c25b731f2317c378efa44f43e8f9bd0e1184ae7be4", size = 3560782, upload-time = "2026-05-01T23:29:11.967Z" },
-    { url = "https://files.pythonhosted.org/packages/09/43/13e9c406fbbf354580476e248a16b64802a376873ebe6339e30bb655572d/psycopg_binary-3.3.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7", size = 4590377, upload-time = "2026-05-01T23:29:18.782Z" },
-    { url = "https://files.pythonhosted.org/packages/22/be/2923cd7c3683e7afdecf4f10796a18de02f5c5ddc0969aa2ad0a8cdd3bbd/psycopg_binary-3.3.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:75a9067e236f9b9ae3535b66fe99bddb33d39c0de10112e49b9ab11eee53dc31", size = 4669023, upload-time = "2026-05-01T23:29:25.884Z" },
-    { url = "https://files.pythonhosted.org/packages/96/a0/2c913d6fe13d6a8bd13597d36739bf47af063ad9399e402cfecab16f3c1e/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b56b603ebcea8aa10b46228b8410ba7f13e7c2ee54389d4d9be0927fd8ce2a70", size = 5467423, upload-time = "2026-05-01T23:29:33.416Z" },
-    { url = "https://files.pythonhosted.org/packages/e7/38/205d10bc1ad0df4a21c5c51659126bd3ea0ef98fcad1e852f78c249bb9c3/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c677c4ad433cb7150c8cd304a0769ae3bcfbe5ea0676eb53faa7b1443b16d0d3", size = 5151137, upload-time = "2026-05-01T23:29:42.013Z" },
-    { url = "https://files.pythonhosted.org/packages/36/fc/f0381ddcd45eff3bb70dbca6823a996048d7f507b2ec3fc92c6fabc0fe87/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26df2717e59c0473e4465a97dfb1b7afebaa479277870fd5784d1436470db47c", size = 6736671, upload-time = "2026-05-01T23:29:51.626Z" },
-    { url = "https://files.pythonhosted.org/packages/95/40/fa545ae152c24327651e5624e4902121e808270be36c10b12e9939be09bc/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dc1f79fd16bb1f3f4421417a514607539f17804d95c7ed617265369d1981cae", size = 4979601, upload-time = "2026-05-01T23:29:56.961Z" },
-    { url = "https://files.pythonhosted.org/packages/86/e4/2f8a47ee97f90cd2b933d0463081d35631ff419de2b8c984a5f369857de0/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:136f199a407b5348b9b857c504aff60c77622a28482e7195839ce1b51238c4cc", size = 4510513, upload-time = "2026-05-01T23:30:07.243Z" },
-    { url = "https://files.pythonhosted.org/packages/0e/0e/94e842ff4a7f98ed162580ca2e8b8864b28c1e0350f2443f8ee47f821167/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b6f5a29e9c775b9f12a1a717aa7a2c80f9e1db6f27ba44a5b59c80ac61d2ffcf", size = 4187243, upload-time = "2026-05-01T23:30:15.352Z" },
-    { url = "https://files.pythonhosted.org/packages/d0/83/fc6c174b672e29b7de996ea77b6cbddf46c891751c3355f6974292baa6b4/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:ee17a2cf4943cde261adfad1bbc5bf38d6b3776d7afff74c7cabcbeaeb08c260", size = 3927347, upload-time = "2026-05-01T23:30:21.186Z" },
-    { url = "https://files.pythonhosted.org/packages/e9/65/768364d4a97a15b1a7f47ba52688c1686f22941d8332a8398cefc468e25f/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c4ab71be17bdca30cb34c34c4e1496e2f5d6f20c199c12bad226070b22ef9bf", size = 4236393, upload-time = "2026-05-01T23:30:26.211Z" },
-    { url = "https://files.pythonhosted.org/packages/bd/3b/218efbc9e645becd80cdf651acda05f85cfe546b7a9c0458c7cbc8fe1f74/psycopg_binary-3.3.4-cp313-cp313-win_amd64.whl", hash = "sha256:dbfdb9b6cc79f31104a7b162a2b921b765fcc62af6c00540a167a8de47e4ed38", size = 3564592, upload-time = "2026-05-01T23:30:31.764Z" },
-    { url = "https://files.pythonhosted.org/packages/48/a6/828c9185701dab71b234c2a76c38a08b098ebfec5020716b4e93807492b5/psycopg_binary-3.3.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:28b7398fdd19db3232c884fb24550bdfe951221f510e195e233299e4c9b78f97", size = 4607292, upload-time = "2026-05-01T23:30:38.962Z" },
-    { url = "https://files.pythonhosted.org/packages/92/58/5b40dbc9d839045c9dae956960e4fb6d20bcabe6c59a2aa34fc3a371913f/psycopg_binary-3.3.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1fbaa292a3c8bb61b45df1ad3da1908ccee7cb889db9425e3557d9e34e2a4829", size = 4687023, upload-time = "2026-05-01T23:30:47.227Z" },
-    { url = "https://files.pythonhosted.org/packages/85/a9/793f0ac107a9003b48441d0d1f9f616d96e0f37458dd8dc12528ceff55fb/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94596f9e7633ee3f6440711d43bb70aa31cc0a46a900ab8b4201a366ace5c9e7", size = 5486985, upload-time = "2026-05-01T23:30:55.517Z" },
-    { url = "https://files.pythonhosted.org/packages/8f/26/42e8533497e2592334f68ec529cf5f840f7fa4e99575a4bb61aa184dbfbf/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8c0056529e68dbe9184cd4019a1f3d8f3a4ead2f6fc7a5afcf27d3314edd1277", size = 5168745, upload-time = "2026-05-01T23:31:01.904Z" },
-    { url = "https://files.pythonhosted.org/packages/15/af/b7151776cc08d5935d45c833ec818a9beb417cf7c08239af1aafbdae78ee/psycopg_binary-3.3.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c09aad7051326e7603c14e50636db9c01f78272dc54b3accff03d46370461e6", size = 6761486, upload-time = "2026-05-01T23:31:14.511Z" },
-    { url = "https://files.pythonhosted.org/packages/d0/ed/c92533b9124712d592cbf1cd6c76da933a2e0acea81dfe1fbe7e735f0cff/psycopg_binary-3.3.4-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:514404ed543efd620c85602b747df2a23cf1241b4067199e1a66f2d2757aaa41", size = 4997427, upload-time = "2026-05-01T23:31:20.901Z" },
-    { url = "https://files.pythonhosted.org/packages/a2/23/ccadfd0de416aa188356daa199453af24087b042e296088706d190ae0295/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:46893c26858be12cc49ca4226ed6a60b4bfccadd946b3bebb783a60b38788228", size = 4533549, upload-time = "2026-05-01T23:31:26.204Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/a0/c8f43cee36386f7bc891ab41a9d31ea07cf9826038e732da79f26b1e5f34/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:df1d567fc430f6df15c9fcf67d87685fc49bdb325adc0db5af1adfb2f44eb5c9", size = 4210256, upload-time = "2026-05-01T23:31:33.884Z" },
-    { url = "https://files.pythonhosted.org/packages/4e/2c/c1547871be3790676e8868b38655496422f94f0978dfb66b74bdba2f1676/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:6b9016b1714da4dd5ecaaa75b82098aa5a0b87854ce9b092e21c27c4ae23e014", size = 3946204, upload-time = "2026-05-01T23:31:39.626Z" },
-    { url = "https://files.pythonhosted.org/packages/c4/b1/f6670f00fa7ea601584623f6c11602ab92117d83eaff885e0210f6de7418/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:47c656a8a7ba6eb0cff1801a4caaa9c8bdc12d03080e273aff1c8ac39971a77e", size = 4255811, upload-time = "2026-05-01T23:31:44.986Z" },
-    { url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" },
-]
-
 [[package]]
 name = "pyarrow"
 version = "23.0.1"
@@ -843,15 +783,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" },
 ]
 
-[[package]]
-name = "tzdata"
-version = "2026.3"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" },
-]
-
 [[package]]
 name = "urllib3"
 version = "2.6.3"

From a760f56e8a6d5e8f3fb660413518cb54299345a1 Mon Sep 17 00:00:00 2001
From: fuziontech <fuziontech@gmail.com>
Date: Tue, 18 Aug 2026 16:56:30 +0000
Subject: [PATCH 3/4] feat(tools): table-scoped per-file index recipes in
 bootstrap-indexes

Adds ducklake_file_column_stats_table_file_idx and
ducklake_file_partition_value_table_file_idx ((table_id, data_file_id)) to
the bootstrap-indexes umbrella, so the tenant maintenance CronJob applies
them fleet-wide on its next pass. These serve the ClickHouse DuckLake
readers table-scoped per-file fetches (WHERE table_id = ? AND data_file_id
IN (...)); the shapes were validated live on megaduck, where they took the
ClickHouse file listing from minutes to seconds. Both names already exist on
megaduck, so adoption there is a no-op.

This replaces the python-subcommand approach previously on this branch:
the justfile bootstrap section (with psql + CONCURRENTLY) is the
established home for catalog index recipes.

Co-authored-by: Shelley <shelley@exe.dev>
---
 tools/justfile | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/tools/justfile b/tools/justfile
index 0672844..2200961 100644
--- a/tools/justfile
+++ b/tools/justfile
@@ -426,9 +426,25 @@ bootstrap-index-file-partition-value-cover: _confirm-target
     @echo ">>> ducklake_file_partition_value_cover_idx"
     @{{ _psql }} -c "CREATE INDEX CONCURRENTLY IF NOT EXISTS ducklake_file_partition_value_cover_idx ON public.ducklake_file_partition_value USING btree (data_file_id, partition_key_index, partition_value);"
 
+# Table-scoped per-file stats fetch: `WHERE table_id = ? AND data_file_id IN
+# (...)`, the shape ClickHouse's DuckLake reader issues when listing a table's
+# files for a query (the catalog-side read behind
+# StorageObjectStorageSource file iteration). Composite so the IN-list
+# prunes inside the index without a table-wide scan.
+[group('bootstrap')]
+bootstrap-index-file-column-stats-table-file: _confirm-target
+    @echo ">>> ducklake_file_column_stats_table_file_idx"
+    @{{ _psql }} -c "CREATE INDEX CONCURRENTLY IF NOT EXISTS ducklake_file_column_stats_table_file_idx ON public.ducklake_file_column_stats USING btree (table_id, data_file_id);"
+
+# Same shape for partition values (ClickHouse partition-pruning fetches).
+[group('bootstrap')]
+bootstrap-index-file-partition-value-table-file: _confirm-target
+    @echo ">>> ducklake_file_partition_value_table_file_idx"
+    @{{ _psql }} -c "CREATE INDEX CONCURRENTLY IF NOT EXISTS ducklake_file_partition_value_table_file_idx ON public.ducklake_file_partition_value USING btree (table_id, data_file_id);"
+
 # Create every DuckLake catalog btree, sequentially (same-relation builds serialize anyway)
 [group('bootstrap')]
-bootstrap-indexes: bootstrap-index-data-file-compaction bootstrap-index-data-file-compaction-order bootstrap-index-data-file-snapshot-read bootstrap-index-delete-file-snapshot-read bootstrap-index-delete-file-table bootstrap-index-delete-file-metrics bootstrap-index-file-column-stats bootstrap-index-file-partition-value-file bootstrap-index-file-partition-value-table bootstrap-index-file-partition-value-cover
+bootstrap-indexes: bootstrap-index-data-file-compaction bootstrap-index-data-file-compaction-order bootstrap-index-data-file-snapshot-read bootstrap-index-delete-file-snapshot-read bootstrap-index-delete-file-table bootstrap-index-delete-file-metrics bootstrap-index-file-column-stats bootstrap-index-file-column-stats-table-file bootstrap-index-file-partition-value-file bootstrap-index-file-partition-value-table bootstrap-index-file-partition-value-table-file bootstrap-index-file-partition-value-cover
 
 # ----------------------------------------------------------------------------
 # state-metrics daemon

From 55b8261e7d76377d3b5e91f4b89a76a6c0556a34 Mon Sep 17 00:00:00 2001
From: fuziontech <fuziontech@gmail.com>
Date: Wed, 19 Aug 2026 21:51:50 +0000
Subject: [PATCH 4/4] =?UTF-8?q?feat(tools):=20ensure-sort-keys=20=E2=80=94?=
 =?UTF-8?q?=20canonical=20DuckLake=20sort=20keys=20for=20the=20tenant=20co?=
 =?UTF-8?q?lumn?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Sets SET SORTED BY (team_id[, timestamp]) on the megaduck tenant tables,
idempotently (skips matching tables, RESETs+SETs changed ones). The
DuckLake extension sorts new writes per file and honors the sort in
compaction merges, so parquet row-group min/max stats become tight for the
tenant column — the ClickHouse DuckLake reader row-group pruning then
skips most of a file for `team_id = X` queries. Before this, every file
spanned the full team_id range (verified live: avg per-file span 563k of
~566k teams) and team filters were full table scans (one OOM-killed a
26Gi pod).

- ensure-sort-keys subcommand (duckdb-side DDL; safe inside the extension
  txn semantics, unlike the CONCURRENTLY index recipes which stay on psql)
- just ensure-sort-keys recipe for the maintenance CronJob chain
- integration tests on a real local DuckLake catalog (apply, idempotence,
  change-of-key RESET, dry-run, canonical-set pin)

Already applied to megaduck imperatively (2026-08-19); this codifies it.
Catalog version stays 1.0 (no reader-compat impact; verified with duckdb
1.5.5, the version viaduck and the maintenance image run).

Co-authored-by: Shelley <shelley@exe.dev>
---
 .../integration/test_sort_keys_integration.py | 73 ++++++++++++++
 tools/ducklake_maintenance.py                 | 96 +++++++++++++++++++
 tools/justfile                                |  9 ++
 3 files changed, 178 insertions(+)
 create mode 100644 tests/integration/test_sort_keys_integration.py

diff --git a/tests/integration/test_sort_keys_integration.py b/tests/integration/test_sort_keys_integration.py
new file mode 100644
index 0000000..8302fbf
--- /dev/null
+++ b/tests/integration/test_sort_keys_integration.py
@@ -0,0 +1,73 @@
+"""ensure-sort-keys: idempotent DuckLake SET SORTED BY maintenance.
+
+The canonical sort keys (ducklake_maintenance.SORT_KEYS) make the extension
+sort new writes per file and honor the sort in compaction merges, so parquet
+row-group min/max stats become tight for the tenant column — ClickHouse's
+row-group pruning then skips most of a file for team_id-filtered queries.
+"""
+
+import ducklake_maintenance
+import pytest
+
+
+def _sort_exprs(conn, table_name):
+    return conn.execute(
+        """
+        SELECT e.sort_key_index, e.expression, e.sort_direction
+        FROM __ducklake_metadata_lake.ducklake_sort_expression e
+        JOIN __ducklake_metadata_lake.ducklake_sort_info i
+          ON i.sort_id = e.sort_id AND i.table_id = e.table_id AND i.end_snapshot IS NULL
+        JOIN __ducklake_metadata_lake.ducklake_table t ON t.table_id = e.table_id
+        WHERE t.table_name = ?
+        ORDER BY e.sort_key_index
+        """,
+        [table_name],
+    ).fetchall()
+
+
+def test_ensure_sort_keys_applies_and_is_idempotent(ducklake_conn):
+    conn = ducklake_conn
+    conn.execute("CREATE TABLE lake.main.t (team_id BIGINT, ts VARCHAR)")
+    conn.execute("INSERT INTO lake.main.t VALUES (1, 'a'), (2, 'b')")
+
+    sort_keys = {("main", "t"): ["team_id"]}
+    applied = ducklake_maintenance.ensure_sort_keys(conn, sort_keys=sort_keys, dry_run=False)
+    assert applied == [("main", "t")]
+    assert _sort_exprs(conn, "t") == [(0, "team_id", "ASC")]
+
+    # second run: nothing to change
+    applied = ducklake_maintenance.ensure_sort_keys(conn, sort_keys=sort_keys, dry_run=False)
+    assert applied == []
+    assert _sort_exprs(conn, "t") == [(0, "team_id", "ASC")]
+
+
+def test_ensure_sort_keys_updates_changed_key(ducklake_conn):
+    conn = ducklake_conn
+    conn.execute("CREATE TABLE lake.main.t (team_id BIGINT, ts VARCHAR)")
+    conn.execute(f"ALTER TABLE lake.main.t SET SORTED BY (ts)")
+
+    applied = ducklake_maintenance.ensure_sort_keys(conn, sort_keys={("main", "t"): ["team_id"]}, dry_run=False)
+    assert applied == [("main", "t")]
+    assert _sort_exprs(conn, "t") == [(0, "team_id", "ASC")]
+
+
+def test_ensure_sort_keys_dry_run_changes_nothing(ducklake_conn):
+    conn = ducklake_conn
+    conn.execute("CREATE TABLE lake.main.t (team_id BIGINT, ts VARCHAR)")
+
+    applied = ducklake_maintenance.ensure_sort_keys(conn, sort_keys={("main", "t"): ["team_id"]}, dry_run=True)
+    assert applied == [("main", "t")]
+    assert _sort_exprs(conn, "t") == []
+
+
+def test_sort_keys_cover_megaduck_tables():
+    """The canonical set must name (schema, table) pairs of the megaduck tenant tables."""
+    expected = {
+        ("main", "events"): ["team_id", "timestamp"],
+        ("main", "events_nrt"): ["team_id", "timestamp"],
+        ("main", "heatmap_events"): ["team_id", "timestamp"],
+        ("main", "person"): ["team_id"],
+        ("main", "person_distinct_id"): ["team_id"],
+        ("main", "groups"): ["team_id"],
+    }
+    assert ducklake_maintenance.SORT_KEYS == expected
diff --git a/tools/ducklake_maintenance.py b/tools/ducklake_maintenance.py
index 579110d..08dd5bb 100644
--- a/tools/ducklake_maintenance.py
+++ b/tools/ducklake_maintenance.py
@@ -1969,6 +1969,88 @@ def _merge_adjacent_call(schema_name: str | None, table_name: str, args: list[st
     )
 
 
+# Canonical DuckLake sort keys: (schema, table) -> sort columns. Applied to
+# megaduck on 2026-08-19; the cron keeps them in place (and applies them to
+# any catalog the routine runs against). The DuckLake extension sorts new
+# writes per file and honors the sort in compaction merges, so parquet
+# row-group min/max stats become tight for the tenant column — the
+# ClickHouse reader's row-group pruning then skips most of a file for
+# team_id-filtered queries (measured: 4/4 row groups reduced to 1 on sorted
+# files; before this, every file's stats spanned the full team_id range and
+# a `team_id = X` query was a full table scan).
+SORT_KEYS: dict[tuple[str, str], list[str]] = {
+    ("main", "events"): ["team_id", "timestamp"],
+    ("main", "events_nrt"): ["team_id", "timestamp"],
+    ("main", "heatmap_events"): ["team_id", "timestamp"],
+    ("main", "person"): ["team_id"],
+    ("main", "person_distinct_id"): ["team_id"],
+    ("main", "groups"): ["team_id"],
+}
+
+
+def _current_sort_keys(conn: duckdb.DuckDBPyConnection) -> dict[tuple[str, str], list[str]]:
+    """(schema, table) -> ordered sort expressions, from the catalog's sort tables."""
+    try:
+        rows = conn.execute(
+            f"""
+            SELECT sch.schema_name, t.table_name, e.expression
+            FROM {METADATA_SCHEMA}.ducklake_sort_expression e
+            JOIN {METADATA_SCHEMA}.ducklake_sort_info i
+              ON i.sort_id = e.sort_id AND i.table_id = e.table_id AND i.end_snapshot IS NULL
+            JOIN {METADATA_SCHEMA}.ducklake_table t ON t.table_id = e.table_id
+            JOIN {METADATA_SCHEMA}.ducklake_schema sch ON sch.schema_id = t.schema_id
+            WHERE t.end_snapshot IS NULL
+            ORDER BY sch.schema_name, t.table_name, e.sort_key_index
+            """
+        ).fetchall()
+    except duckdb.CatalogException:
+        # Catalog predates sorted tables (no ducklake_sort_* tables yet).
+        return {}
+    current: dict[tuple[str, str], list[str]] = {}
+    for schema_name, table_name, expression in rows:
+        current.setdefault((schema_name, table_name), []).append(expression)
+    return current
+
+
+def ensure_sort_keys(
+    conn: duckdb.DuckDBPyConnection,
+    sort_keys: dict[tuple[str, str], list[str]] | None = None,
+    dry_run: bool = False,
+) -> list[tuple[str, str]]:
+    """Apply SORT_KEYS to the catalog, idempotently. Returns the tables (re)applied.
+
+    A table whose current sort key already matches is skipped; a table with a
+    different sort key is re-set (SET SORTED BY replaces the sort spec).
+    Dry-run reports what would change without touching the catalog.
+    """
+    wanted = SORT_KEYS if sort_keys is None else sort_keys
+    current = _current_sort_keys(conn)
+    applied: list[tuple[str, str]] = []
+    for (schema_name, table_name), columns in sorted(wanted.items()):
+        existing = current.get((schema_name, table_name))
+        if existing == columns:
+            log.info("ensure-sort-keys: %s.%s already sorted by (%s), skipping", schema_name, table_name, ", ".join(columns))
+            continue
+        key_sql = ", ".join(f'"{c}"' if not c.isidentifier() else c for c in columns)
+        log.info(
+            "ensure-sort-keys: %s %s.%s SET SORTED BY (%s)%s",
+            "would apply" if dry_run else "applying",
+            schema_name,
+            table_name,
+            ", ".join(columns),
+            " (dry run)" if dry_run else "",
+        )
+        if not dry_run:
+            if existing:
+                # SET SORTED BY appends a new sort spec; it does not end the old one.
+                conn.execute(f'ALTER TABLE {ATTACH_NAME}."{schema_name}"."{table_name}" RESET SORTED BY')
+            conn.execute(
+                f'ALTER TABLE {ATTACH_NAME}."{schema_name}"."{table_name}" SET SORTED BY ({key_sql})'
+            )
+        applied.append((schema_name, table_name))
+    return applied
+
+
 def compact(
     conn: duckdb.DuckDBPyConnection,
     tier: int,
@@ -2422,6 +2504,18 @@ def build_parser() -> argparse.ArgumentParser:
     p.add_argument("--days", type=int, default=7)
     p.add_argument("--dry-run", action="store_true")
 
+    # ensure-sort-keys
+    p = sub.add_parser(
+        "ensure-sort-keys",
+        help="Apply the canonical DuckLake sort keys (SORT_KEYS) idempotently",
+        description=(
+            "Apply the canonical sort keys so new writes and compaction outputs are "
+            "sorted by the tenant column (tight parquet row-group stats -> ClickHouse "
+            "row-group pruning on team_id filters). Idempotent; safe to run every pass."
+        ),
+    )
+    p.add_argument("--dry-run", action="store_true")
+
     # checkpoint
     sub.add_parser("checkpoint", help="CHECKPOINT (merge + expire + cleanup)")
 
@@ -2556,6 +2650,8 @@ def main(argv: list[str] | None = None) -> None:
                 fsck(conn, args.dry_run, args.max_iterations)
             case "orphans":
                 orphans(conn, args.dry_run)
+            case "ensure-sort-keys":
+                ensure_sort_keys(conn, dry_run=args.dry_run)
             case "maintain":
                 maintain(conn, args.days, args.dry_run)
             case "checkpoint":
diff --git a/tools/justfile b/tools/justfile
index 2200961..a7fbe2f 100644
--- a/tools/justfile
+++ b/tools/justfile
@@ -446,6 +446,15 @@ bootstrap-index-file-partition-value-table-file: _confirm-target
 [group('bootstrap')]
 bootstrap-indexes: bootstrap-index-data-file-compaction bootstrap-index-data-file-compaction-order bootstrap-index-data-file-snapshot-read bootstrap-index-delete-file-snapshot-read bootstrap-index-delete-file-table bootstrap-index-delete-file-metrics bootstrap-index-file-column-stats bootstrap-index-file-column-stats-table-file bootstrap-index-file-partition-value-file bootstrap-index-file-partition-value-table bootstrap-index-file-partition-value-table-file bootstrap-index-file-partition-value-cover
 
+# Apply the canonical DuckLake sort keys (tenant column first on every
+# tenant table): new writes and compaction outputs come out sorted, giving
+# tight parquet row-group stats, which is what the ClickHouse reader's
+# row-group pruning needs for team_id-filtered queries. Idempotent.
+[group('bootstrap')]
+ensure-sort-keys: _confirm-target
+    @echo ">>> ensure-sort-keys"
+    @env {{ _target_env }} python {{ ducklake_maintenance }} ensure-sort-keys
+
 # ----------------------------------------------------------------------------
 # state-metrics daemon
 # ----------------------------------------------------------------------------