From fccffe69432828c573fcbd62747fdc7ac3f0ed1c Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 8 Jul 2026 18:33:20 -0400 Subject: [PATCH 1/5] perf: Route count_overlaps LEFT-join through IE_JOIN with zero-fill The idiomatic per-interval overlap count -- SELECT a.cols, COUNT(b.col) FROM a LEFT JOIN b ON a.interval INTERSECTS b.interval GROUP BY a.cols -- declined to the naive plan like every outer join, becoming a HASH_JOIN on the low-cardinality chromosome key with the position inequalities as a residual filter (quadratic; ~180x slower than achievable at scale). Detect this shape and reuse the INNER IEJoin path (which already plans COUNT + GROUP BY through IE_JOIN), then wrap its counts in a zero-fill LEFT join back onto the distinct left keys so left intervals with no overlap report 0. Supports a single COUNT() / COUNT(DISTINCT ) with a GROUP BY on the projected left columns; COUNT(*), non-COUNT aggregates, ORDER BY / LIMIT / HAVING, and other outer-join shapes still take the naive-predicate plan. Claude-Session: https://claude.ai/code/session_01TERWBHov76DQM3nQeT8yyy --- src/giql/expanders/intersects_duckdb.py | 185 ++++++++++++++++++++++++ 1 file changed, 185 insertions(+) diff --git a/src/giql/expanders/intersects_duckdb.py b/src/giql/expanders/intersects_duckdb.py index 02d1afd..ce21fee 100644 --- a/src/giql/expanders/intersects_duckdb.py +++ b/src/giql/expanders/intersects_duckdb.py @@ -407,6 +407,132 @@ def from_intersects( ) +@dataclass(frozen=True) +class _CountOverlapsShape: + """The matched ``count_overlaps`` LEFT-join-with-COUNT shape (#209). + + Bundles the LEFT :class:`~sqlglot.expressions.Join`, the un-aliased left + (FROM) base table, the left/right user aliases (case-folded), the left-side + group-key SELECT items, and the single ``COUNT`` aggregate SELECT item. The + emission reuses the INNER path to build the per-chromosome IEJoin count and + wraps it in a zero-fill LEFT join against the distinct left keys. + """ + + the_join: exp.Join + left_table: exp.Table + left_alias: str + right_alias: str + key_items: tuple[exp.Expression, ...] + agg_item: exp.Alias + + +def _match_count_overlaps(query: exp.Expression) -> "_CountOverlapsShape | None": + """Return the count_overlaps shape when *query* matches it, else ``None``. + + Matches exactly ``SELECT , COUNT() AS FROM + a LEFT JOIN b ON GROUP BY + `` -- the only right-side reference is the COUNT argument, and the + projected left columns are the group keys. Every other shape (``COUNT(*)``, + non-COUNT or multiple aggregates, a WHERE / HAVING / ORDER BY / LIMIT, a + non-LEFT outer join, extra ON residuals) returns ``None`` so the caller keeps + the existing behaviour (the naive-predicate plan). + """ + if not isinstance(query, exp.Select): + return None + if query.args.get("with_") or query.args.get("distinct"): + return None + if any( + query.args.get(k) is not None for k in ("having", "order", "limit", "offset") + ): + return None + if query.args.get("group") is None or query.args.get("where") is not None: + return None + + joins = query.args.get("joins") or [] + if len(joins) != 1: + return None + the_join = joins[0] + if _normalize_alias(the_join.args.get("side") or "") != "left": + return None + if (the_join.args.get("kind") or "").upper() not in ("", "OUTER"): + return None + on = the_join.args.get("on") + if on is None: + return None + intersects = _find_column_intersects_in(on) + if intersects is None or _count_column_intersects(query) != 1: + return None + # v1 supports only a bare INTERSECTS ON (no residual join conditions). + if _strip_intersects(on, intersects) is not None: + return None + + from_node = query.args.get("from_") + from_table = from_node.this if from_node else None + join_table = the_join.this + if not isinstance(from_table, exp.Table) or not from_table.name: + return None + if not isinstance(join_table, exp.Table) or not join_table.name: + return None + left_alias = _normalize_alias(from_table.alias_or_name) + right_alias = _normalize_alias(join_table.alias_or_name) + if left_alias == right_alias: + return None + + key_items: list[exp.Expression] = [] + agg_item: exp.Alias | None = None + for sel in query.expressions: + target = sel.this if isinstance(sel, exp.Alias) else sel + while isinstance(target, exp.Paren): + target = target.this + if isinstance(target, exp.AggFunc): + # Exactly one COUNT() / COUNT(DISTINCT ) aggregate, + # aliased so the wrapper can reference and re-project it. + if agg_item is not None or not isinstance(target, exp.Count): + return None + if not isinstance(sel, exp.Alias) or not sel.output_name: + return None + arg = target.this + if isinstance(arg, exp.Distinct): + if len(arg.expressions) != 1: + return None + arg = arg.expressions[0] + if not ( + isinstance(arg, exp.Column) + and not isinstance(arg.this, exp.Star) + and _normalize_alias(arg.table) == right_alias + ): + return None + agg_item = sel + elif ( + isinstance(target, exp.Column) + and target.table + and not isinstance(target.this, exp.Star) + ): + if _normalize_alias(target.table) != left_alias: + return None + key_items.append(sel) + else: + return None + + if agg_item is None or not key_items: + return None + # Every projected key must be a group key (guard SELECT-only left columns). + group_sql = {e.sql(dialect="duckdb") for e in query.args["group"].expressions} + for item in key_items: + expr = item.this if isinstance(item, exp.Alias) else item + if expr.sql(dialect="duckdb") not in group_sql: + return None + + return _CountOverlapsShape( + the_join=the_join, + left_table=from_table, + left_alias=left_alias, + right_alias=right_alias, + key_items=tuple(key_items), + agg_item=agg_item, + ) + + class IntersectsDuckDBIEJoinTransformer: """Transform column-to-column INTERSECTS joins into a DuckDB IEJoin pattern. @@ -508,6 +634,15 @@ def transform_to_sql(self, query: exp.Expression) -> str | None: if distinct_node is not None and distinct_node.args.get("on"): return None + # count_overlaps fast path (#209): a LEFT JOIN whose only right-side use + # is a COUNT(b.col) with a GROUP BY on the left keys reaches IE_JOIN via + # the INNER path plus a zero-fill LEFT join, rather than declining to the + # naive HASH_JOIN + inequality filter that every other outer join takes. + # Checked before the outer-join decline below since it *is* an outer join. + count_shape = _match_count_overlaps(query) + if count_shape is not None: + return self._build_count_overlaps_sql(query, count_shape) + if _has_outer_join_intersects(query): return None @@ -653,6 +788,56 @@ def transform_to_sql(self, query: exp.Expression) -> str | None: except _UnqualifiedProjectionError as exc: raise ValueError(str(exc)) from exc + def _build_count_overlaps_sql( + self, query: exp.Select, shape: "_CountOverlapsShape" + ) -> str | None: + """Render the count_overlaps zero-fill SQL for a matched LEFT-join shape (#209). + + Reuses the INNER IEJoin path: an INNER copy of the query (same + projections and GROUP BY, LEFT downgraded to INNER) is transpiled by + :meth:`transform_to_sql` into ``SET VARIABLE ...; ``, which + DuckDB plans through ``IE_JOIN`` + hash aggregate. The INNER count drops + left rows with no overlap, so its ``SELECT`` is wrapped as a CTE and + LEFT-joined back onto the distinct left keys, zero-filling the missing + counts. Returns ``None`` (fall back to the naive plan) if the INNER copy + itself declines. + """ + inner_query = query.copy() + inner_join = inner_query.args["joins"][0] + inner_join.set("side", None) + inner_join.set("kind", None) + inner_sql = self.transform_to_sql(inner_query) + if inner_sql is None or ";\n" not in inner_sql: + return None + set_var_stmt, inner_select = inner_sql.split(";\n", 1) + + q = self._sql_quote_ident + key_names = [item.output_name for item in shape.key_items] + key_projections = [item.sql(dialect="duckdb") for item in shape.key_items] + group_exprs = [ + (item.this if isinstance(item, exp.Alias) else item).sql(dialect="duckdb") + for item in shape.key_items + ] + agg_name = shape.agg_item.output_name + + # Distinct left-key relation, rendering the FROM table with its own alias + # so the key projections (``a.chrom`` ...) resolve against it. + base_cte = ( + f"SELECT {', '.join(key_projections)} " + f"FROM {shape.left_table.sql(dialect='duckdb')} " + f"GROUP BY {', '.join(group_exprs)}" + ) + using_cols = ", ".join(q(name) for name in key_names) + base_out = ", ".join(f"base.{q(name)}" for name in key_names) + wrapper = ( + f"WITH __giql_counts AS ({inner_select}), " + f"__giql_base AS ({base_cte}) " + f"SELECT {base_out}, COALESCE(c.{q(agg_name)}, 0) AS {q(agg_name)} " + f"FROM __giql_base AS base " + f"LEFT JOIN __giql_counts AS c USING ({using_cols})" + ) + return set_var_stmt + ";\n" + wrapper + @staticmethod def _sql_escape(s: str) -> str: """Return *s* with single quotes doubled for safe SQL string literal use.""" From 8dc88f827958d365f6263d8300757e27db8f9555 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 8 Jul 2026 18:33:20 -0400 Subject: [PATCH 2/5] test: Cover the count_overlaps IE_JOIN zero-fill fast path Add shape, decline (COUNT(*), non-COUNT aggregate, no GROUP BY, ORDER BY, self-join), a Hypothesis property test against a Python count_overlaps reference (zero-overlap rows, one-sided chromosomes, empty right table, duplicate left rows), and an EXPLAIN plan guard asserting the generated inner SQL is not planned as a BLOCKWISE_NL_JOIN. Claude-Session: https://claude.ai/code/session_01TERWBHov76DQM3nQeT8yyy --- tests/test_duckdb_iejoin.py | 281 ++++++++++++++++++++++++++++++++++++ 1 file changed, 281 insertions(+) diff --git a/tests/test_duckdb_iejoin.py b/tests/test_duckdb_iejoin.py index e399f67..5f69e1c 100644 --- a/tests/test_duckdb_iejoin.py +++ b/tests/test_duckdb_iejoin.py @@ -63,6 +63,24 @@ def _python_anti_overlap(peaks: list[tuple], genes: list[tuple]) -> list[tuple]: return sorted({(pc, ps, pe) for (pc, ps, pe) in peaks} - matched) +def _python_count_overlaps(peaks: list[tuple], genes: list[tuple]) -> list[tuple]: + """Reference count_overlaps as ``(chrom, start, end, count)`` per distinct left key. + + Mirrors ``COUNT(b.col)`` under ``GROUP BY`` on the left keys: each distinct + left interval's count is the number of overlapping right intervals summed + across the duplicate left rows sharing that key (so a key appearing twice + with one overlap counts 2, matching SQL group-by-over-a-LEFT-join). Left keys + with no overlap are present with count 0. + """ + out: list[tuple] = [] + for key in set(peaks): + pc, ps, pe = key + duplicates = sum(1 for p in peaks if p == key) + b_overlaps = sum(1 for (gc, gs, ge) in genes if pc == gc and pe > gs and ge > ps) + out.append((pc, ps, pe, duplicates * b_overlaps)) + return sorted(out) + + def _explain_dynamic_sql(conn, sql: str) -> str: """Return the DuckDB EXPLAIN text of the per-chromosome dynamic SQL for *sql*. @@ -3198,6 +3216,269 @@ def test_transpile_should_reject_right_side_projection_when_join_is_anti(self): assert "left-side" in str(excinfo.value) +class TestTranspileDuckDBIEJoinCountOverlaps: + """The count_overlaps fast path: LEFT JOIN + COUNT(b.col) + GROUP BY (#209).""" + + def test_transpile_should_emit_zero_fill_wrapper_when_left_join_count(self): + """Test that a LEFT-join COUNT over INTERSECTS emits the zero-fill fast path. + + Given: + A ``SELECT a.cols, COUNT(b.col) ... FROM a LEFT JOIN b ON + a.interval INTERSECTS b.interval GROUP BY a.cols`` query. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should emit the INNER IEJoin count wrapped in a zero-fill + LEFT join (a ``SET VARIABLE`` block, a ``__giql_counts`` CTE, and + a ``COALESCE(..., 0)``), never the naive ``a.chrom = b.chrom`` + hash-join predicate. + """ + # Arrange + query = ( + 'SELECT a.chrom, a.start, a."end", COUNT(b.chrom) AS n ' + "FROM peaks a LEFT JOIN genes b ON a.interval INTERSECTS b.interval " + 'GROUP BY a.chrom, a.start, a."end"' + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert "__giql_counts" in sql + assert "COALESCE" in sql + assert 'a."chrom" = b."chrom"' not in sql + + def test_transpile_should_decline_count_overlaps_when_count_star(self): + """Test that COUNT(*) over a LEFT join declines to the naive plan. + + Given: + A LEFT-join query aggregating ``COUNT(*)`` rather than a + right-side column. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should decline the fast path (no ``__giql_counts`` wrapper), + because COUNT(*) over a LEFT join counts the null-padded row for + unmatched left rows, which the zero-fill rewrite cannot express. + """ + # Arrange + query = ( + "SELECT a.chrom, COUNT(*) AS n " + "FROM a LEFT JOIN b ON a.interval INTERSECTS b.interval GROUP BY a.chrom" + ) + + # Act + sql = transpile(query, tables=["a", "b"], dialect="duckdb") + + # Assert + assert "__giql_counts" not in sql + + def test_transpile_should_decline_count_overlaps_when_non_count_aggregate(self): + """Test that a non-COUNT aggregate over a LEFT join declines to the naive plan. + + Given: + A LEFT-join query aggregating ``SUM(b.score)``. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should decline the fast path (no ``__giql_counts`` wrapper); + the fast path supports only ``COUNT`` in v1. + """ + # Arrange + query = ( + "SELECT a.chrom, SUM(b.score) AS s " + "FROM a LEFT JOIN b ON a.interval INTERSECTS b.interval GROUP BY a.chrom" + ) + + # Act + sql = transpile(query, tables=["a", "b"], dialect="duckdb") + + # Assert + assert "__giql_counts" not in sql + + def test_transpile_should_decline_count_overlaps_when_no_group_by(self): + """Test that a LEFT-join COUNT without GROUP BY declines to the naive plan. + + Given: + A LEFT-join COUNT query with no GROUP BY clause. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should decline the fast path (no ``__giql_counts`` wrapper). + """ + # Arrange + query = ( + "SELECT COUNT(b.chrom) AS n " + "FROM a LEFT JOIN b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["a", "b"], dialect="duckdb") + + # Assert + assert "__giql_counts" not in sql + + def test_transpile_should_decline_count_overlaps_when_order_by(self): + """Test that a LEFT-join COUNT with ORDER BY declines to the naive plan. + + Given: + A LEFT-join COUNT query carrying a top-level ORDER BY. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should decline the fast path (no ``__giql_counts`` wrapper); + result-ordering clauses are out of scope for v1. + """ + # Arrange + query = ( + "SELECT a.chrom, COUNT(b.chrom) AS n " + "FROM a LEFT JOIN b ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom ORDER BY a.chrom" + ) + + # Act + sql = transpile(query, tables=["a", "b"], dialect="duckdb") + + # Assert + assert "__giql_counts" not in sql + + def test_transpile_should_decline_count_overlaps_when_self_join(self): + """Test that a self-join LEFT COUNT declines to the naive plan. + + Given: + A LEFT-join COUNT whose two sides are the same table (a self + overlap count). + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should decline the fast path (no ``__giql_counts`` wrapper), + because the reused INNER path declines the self-join and the + count builder falls back to the naive plan. + """ + # Arrange + query = ( + "SELECT a.chrom, COUNT(b.chrom) AS n " + "FROM t a LEFT JOIN t b ON a.interval INTERSECTS b.interval GROUP BY a.chrom" + ) + + # Act + sql = transpile(query, tables=["t"], dialect="duckdb") + + # Assert + assert "__giql_counts" not in sql + + @settings( + max_examples=25, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given( + peaks=st.lists( + st.tuples( + st.sampled_from(["chr1", "chr2", "chr3"]), + st.integers(min_value=0, max_value=100), + st.integers(min_value=1, max_value=30), + ), + min_size=0, + max_size=8, + ), + genes=st.lists( + st.tuples( + st.sampled_from(["chr1", "chr2", "chr3"]), + st.integers(min_value=0, max_value=100), + st.integers(min_value=1, max_value=30), + ), + min_size=0, + max_size=8, + ), + ) + def test_count_overlaps_should_match_python_reference_for_random_inputs( + self, conn, peaks, genes + ): + """Test the count_overlaps fast path against a Python-native reference. + + Given: + A Hypothesis-generated pair of small interval lists. + When: + A ``COUNT(b.chrom)`` LEFT-join over INTERSECTS is transpiled with + ``dialect='duckdb'`` and executed. + Then: + The returned ``(chrom, start, end, count)`` rows should equal the + Python count_overlaps reference, including left rows with no + overlap (count 0), chromosomes present on only one side, an empty + right table, and duplicate left rows. + """ + # Arrange + peak_rows = [(c, s, s + length) for (c, s, length) in peaks] + gene_rows = [(c, s, s + length) for (c, s, length) in genes] + conn.execute("DROP TABLE IF EXISTS peaks") + conn.execute("DROP TABLE IF EXISTS genes") + conn.execute( + 'CREATE TABLE peaks (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' + ) + if peak_rows: + conn.executemany("INSERT INTO peaks VALUES (?, ?, ?)", peak_rows) + conn.execute( + 'CREATE TABLE genes (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' + ) + if gene_rows: + conn.executemany("INSERT INTO genes VALUES (?, ?, ?)", gene_rows) + sql = transpile( + 'SELECT a.chrom, a.start, a."end", COUNT(b.chrom) AS n ' + "FROM peaks a LEFT JOIN genes b ON a.interval INTERSECTS b.interval " + 'GROUP BY a.chrom, a.start, a."end"', + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + expected = _python_count_overlaps(peak_rows, gene_rows) + + # Assert + assert rows == expected + + def test_count_overlaps_plan_should_avoid_nested_loop_when_executed_on_duckdb( + self, conn + ): + """Test that the count_overlaps fast path is not planned as a nested loop. + + Given: + A dense single-chromosome peaks/genes pair large enough that a + naive LEFT-join count would be planned as a quadratic + ``BLOCKWISE_NL_JOIN``. + When: + The ``dialect='duckdb'`` count query is generated and its inner + per-chromosome dynamic SQL is planned with ``EXPLAIN``. + Then: + It should avoid ``BLOCKWISE_NL_JOIN`` — the INNER IEJoin count + routes through DuckDB's range-join family (#209). + """ + # Arrange + _make_table( + conn, "peaks", [("chr1", i * 7, i * 7 + 5, "p", 1, "+") for i in range(4000)] + ) + _make_table( + conn, + "genes", + [("chr1", i * 7 + 2, i * 7 + 9, "g", 1, "+") for i in range(4000)], + ) + sql = transpile( + 'SELECT a.chrom, a.start, a."end", COUNT(b.chrom) AS n ' + "FROM peaks a LEFT JOIN genes b ON a.interval INTERSECTS b.interval " + 'GROUP BY a.chrom, a.start, a."end"', + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + plan = _explain_dynamic_sql(conn, sql) + + # Assert + assert "BLOCKWISE_NL_JOIN" not in plan + + class TestTranspileDuckDBIEJoinExecution: """End-to-end execution tests against in-memory DuckDB.""" From 3324f022cdb6dce86c3e308c2801158ae7da00a3 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 8 Jul 2026 18:33:20 -0400 Subject: [PATCH 3/5] docs: Document the count_overlaps fast path in the performance guide Describe the LEFT-join COUNT over INTERSECTS routing through the INNER IE_JOIN count plus a zero-fill LEFT join, and the shapes that remain on the naive-predicate plan. Claude-Session: https://claude.ai/code/session_01TERWBHov76DQM3nQeT8yyy --- docs/transpilation/performance.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/transpilation/performance.rst b/docs/transpilation/performance.rst index cdcf18d..1fcd643 100644 --- a/docs/transpilation/performance.rst +++ b/docs/transpilation/performance.rst @@ -410,6 +410,20 @@ path — the correlated form reaches the fast path while preserving exact semantics (#208). Expect large speedups over the naive predicate for all three variants at scale. +**count_overlaps.** The idiomatic per-interval overlap count — ``SELECT +a.cols, COUNT(b.col) FROM a LEFT JOIN b ON a.interval INTERSECTS +b.interval GROUP BY a.cols`` — is also accelerated (#209). A plain +``LEFT JOIN`` inequality overlap would decline to the naive predicate (a +``HASH_JOIN`` on ``chrom`` with the position inequalities as a residual +filter, quadratic when the chromosome key has low cardinality). Instead +the dialect computes the counts through the INNER ``IE_JOIN`` path and +wraps them in a zero-fill ``LEFT JOIN`` back onto the distinct left keys, +so left intervals with no overlap report ``0``. This covers a single +``COUNT()`` (or ``COUNT(DISTINCT )``) with a +``GROUP BY`` on the projected left columns; ``COUNT(*)``, non-COUNT +aggregates, ``ORDER BY`` / ``LIMIT`` / ``HAVING``, and other outer-join +shapes still take the naive-predicate plan. + On top of the core shape the dialect also absorbs several common decorations: From bb07b199589fedc62f6570a6e706213e92927f06 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 8 Jul 2026 18:47:33 -0400 Subject: [PATCH 4/5] fix: Decline count_overlaps shapes that diverge from the naive plan Tighten the count_overlaps detection gate for two shapes a review found producing wrong results on matched queries: a GROUP BY carrying keys beyond the projected columns (the extra key inflates the count cardinality and drops zero-count rows the zero-fill base cannot reproduce), and a projected key whose output name collides with the COUNT alias (the zero-fill USING / COALESCE binds to the wrong duplicately-named column). Require the group keys to equal the projected keys exactly and all output names to be distinct, declining to the naive plan otherwise. Claude-Session: https://claude.ai/code/session_01TERWBHov76DQM3nQeT8yyy --- src/giql/expanders/intersects_duckdb.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/giql/expanders/intersects_duckdb.py b/src/giql/expanders/intersects_duckdb.py index ce21fee..126d87d 100644 --- a/src/giql/expanders/intersects_duckdb.py +++ b/src/giql/expanders/intersects_duckdb.py @@ -516,12 +516,24 @@ def _match_count_overlaps(query: exp.Expression) -> "_CountOverlapsShape | None" if agg_item is None or not key_items: return None - # Every projected key must be a group key (guard SELECT-only left columns). - group_sql = {e.sql(dialect="duckdb") for e in query.args["group"].expressions} - for item in key_items: - expr = item.this if isinstance(item, exp.Alias) else item - if expr.sql(dialect="duckdb") not in group_sql: - return None + # The GROUP BY keys must be exactly the projected left columns. An extra + # group key produces more rows than the distinct projected keys (which the + # zero-fill base relation reconstructs and so cannot reproduce), and a + # projected key absent from the group would be an invalid aggregate query. + projected_key_exprs = { + (item.this if isinstance(item, exp.Alias) else item).sql(dialect="duckdb") + for item in key_items + } + group_exprs = {e.sql(dialect="duckdb") for e in query.args["group"].expressions} + if group_exprs != projected_key_exprs: + return None + # The zero-fill wrapper joins on the user-facing output names and re-projects + # the count under its alias, so those names must be distinct — otherwise a + # key column and the count alias collide and USING / COALESCE bind to the + # wrong column. + output_names = [item.output_name for item in key_items] + [agg_item.output_name] + if len(set(output_names)) != len(output_names): + return None return _CountOverlapsShape( the_join=the_join, From 6ebea6932d7339c3019174738e8f15b08fb58f4e Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 8 Jul 2026 18:47:33 -0400 Subject: [PATCH 5/5] test: Cover count_overlaps decline for group/projection and collisions Add decline tests for a GROUP BY exceeding the projection, an output-name collision between a key and the COUNT alias, and a top-level WHERE, plus a COUNT(DISTINCT b.col) execution test asserting equivalence with the naive plan. Claude-Session: https://claude.ai/code/session_01TERWBHov76DQM3nQeT8yyy --- tests/test_duckdb_iejoin.py | 124 ++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/tests/test_duckdb_iejoin.py b/tests/test_duckdb_iejoin.py index 5f69e1c..f09f736 100644 --- a/tests/test_duckdb_iejoin.py +++ b/tests/test_duckdb_iejoin.py @@ -3368,6 +3368,130 @@ def test_transpile_should_decline_count_overlaps_when_self_join(self): # Assert assert "__giql_counts" not in sql + def test_transpile_should_decline_count_overlaps_when_group_by_exceeds_projection( + self, + ): + """Test that a GROUP BY with keys beyond the projection declines to naive. + + Given: + A LEFT-join COUNT that groups by ``a.chrom, a.start`` but projects + only ``a.chrom``. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should decline the fast path (no ``__giql_counts`` wrapper), + because the extra group key produces more rows than the distinct + projected keys, which the zero-fill base cannot reproduce. + """ + # Arrange + query = ( + "SELECT a.chrom, COUNT(b.chrom) AS n " + "FROM a LEFT JOIN b ON a.interval INTERSECTS b.interval " + 'GROUP BY a.chrom, a."start"' + ) + + # Act + sql = transpile(query, tables=["a", "b"], dialect="duckdb") + + # Assert + assert "__giql_counts" not in sql + + def test_transpile_should_decline_count_overlaps_when_output_names_collide(self): + """Test that a key/count output-name collision declines to naive. + + Given: + A LEFT-join COUNT whose projected key column and count alias share + the output name ``n``. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should decline the fast path (no ``__giql_counts`` wrapper), + because the zero-fill USING / COALESCE would bind to the wrong + duplicately-named column. + """ + # Arrange + query = ( + "SELECT a.n, COUNT(b.chrom) AS n " + "FROM a LEFT JOIN b ON a.interval INTERSECTS b.interval GROUP BY a.n" + ) + + # Act + sql = transpile(query, tables=["a", "b"], dialect="duckdb") + + # Assert + assert "__giql_counts" not in sql + + def test_transpile_should_decline_count_overlaps_when_where_present(self): + """Test that a top-level WHERE declines the count fast path to naive. + + Given: + A LEFT-join COUNT carrying a top-level ``WHERE`` filter. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should decline the fast path (no ``__giql_counts`` wrapper); + post-join filters are out of scope for v1. + """ + # Arrange + query = ( + "SELECT a.chrom, COUNT(b.chrom) AS n " + "FROM a LEFT JOIN b ON a.interval INTERSECTS b.interval " + "WHERE a.start >= 0 GROUP BY a.chrom" + ) + + # Act + sql = transpile(query, tables=["a", "b"], dialect="duckdb") + + # Assert + assert "__giql_counts" not in sql + + def test_count_overlaps_should_match_naive_plan_when_count_distinct(self, conn): + """Test that COUNT(DISTINCT b.col) matches the naive plan when executed. + + Given: + Peaks and genes where a peak overlaps multiple genes sharing a + name (so DISTINCT collapses them) plus a non-overlapping peak. + When: + A ``COUNT(DISTINCT b.name)`` LEFT-join fast path is executed. + Then: + The rows should equal the naive-predicate plan, including the + zero-filled non-overlapping peak. + """ + # Arrange + conn.execute( + 'CREATE TABLE peaks (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' + ) + conn.execute( + "CREATE TABLE genes " + '(chrom VARCHAR, "start" INTEGER, "end" INTEGER, name VARCHAR)' + ) + conn.executemany( + "INSERT INTO peaks VALUES (?, ?, ?)", + [("chr1", 100, 200), ("chr1", 900, 950)], + ) + conn.executemany( + "INSERT INTO genes VALUES (?, ?, ?, ?)", + [ + ("chr1", 120, 130, "g1"), + ("chr1", 140, 150, "g1"), + ("chr1", 160, 170, "g2"), + ], + ) + query = ( + 'SELECT a.chrom, a.start, a."end", COUNT(DISTINCT b.name) AS n ' + "FROM peaks a LEFT JOIN genes b ON a.interval INTERSECTS b.interval " + 'GROUP BY a.chrom, a.start, a."end"' + ) + fast_sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + naive_sql = transpile(query, tables=["peaks", "genes"]) + + # Act + fast_rows = sorted(conn.execute(fast_sql).fetchall()) + naive_rows = sorted(conn.execute(naive_sql).fetchall()) + + # Assert + assert fast_rows == naive_rows + @settings( max_examples=25, deadline=None,