diff --git a/docs/transpilation/performance.rst b/docs/transpilation/performance.rst index 6b6721d..cdcf18d 100644 --- a/docs/transpilation/performance.rst +++ b/docs/transpilation/performance.rst @@ -399,13 +399,16 @@ left-distinct chromosome partition rather than the chromosome INTERSECT used by INNER / SEMI, so left rows on chromosomes absent from the right table are preserved. -DuckDB plans INNER through the IE_JOIN / PIECEWISE_MERGE_JOIN -sort-merge family. SEMI and ANTI with inequality predicates plan -through ``BLOCKWISE_NL_JOIN`` instead — not the IE_JOIN sort-merge fast -path, but still a per-chromosome plan. Expect speedups vs. the naive -predicate for SEMI / ANTI -where the chromosome partition already filters most pairs; INNER gets -the largest speedup. +DuckDB plans all three variants through its ``IE_JOIN`` / +``PIECEWISE_MERGE_JOIN`` sort-merge range-join family. INNER emits a +per-chromosome inner join directly. SEMI and ANTI are emitted as a +correlated ``WHERE EXISTS`` / ``WHERE NOT EXISTS`` subquery over the +per-chromosome right partition, because DuckDB plans a bare ``SEMI +JOIN`` / ``ANTI JOIN`` inequality overlap as a quadratic +``BLOCKWISE_NL_JOIN`` (a nested loop) rather than the range-join fast +path — the correlated form reaches the fast path while preserving exact +semantics (#208). Expect large speedups over the naive predicate for all +three variants at scale. On top of the core shape the dialect also absorbs several common decorations: diff --git a/src/giql/expanders/intersects_duckdb.py b/src/giql/expanders/intersects_duckdb.py index 6514c7d..02d1afd 100644 --- a/src/giql/expanders/intersects_duckdb.py +++ b/src/giql/expanders/intersects_duckdb.py @@ -414,16 +414,24 @@ class IntersectsDuckDBIEJoinTransformer: 1. ``SET VARIABLE __giql_iejoin_ = COALESCE(, );`` where ```` is a ``string_agg`` over the chromosome - partition that emits one ``SELECT ... FROM (... WHERE chrom = '') - a JOIN (... WHERE chrom = '') b ON `` per - chromosome, joined by ``UNION ALL``. + partition that emits one per-chromosome overlap query joined by + ``UNION ALL``. For INNER, that is ``SELECT ... FROM (... WHERE chrom = + '') a JOIN (... WHERE chrom = '') b ON ``; for + SEMI / ANTI it is ``SELECT ... FROM (... WHERE chrom = '') a WHERE + [NOT] EXISTS (SELECT 1 FROM (... WHERE chrom = '') b WHERE + )``. 2. ``SELECT FROM query(getvariable('__giql_iejoin_'))`` Each per-chromosome subquery uses pure inequality predicates which DuckDB plans through its range-join family (``IE_JOIN`` or ``PIECEWISE_MERGE_JOIN``). - The dialect supports INNER, SEMI, and ANTI variants — the per-chromosome - join keyword switches accordingly, and SEMI / ANTI restrict the outer - SELECT to left-side projections only. + The dialect supports INNER, SEMI, and ANTI variants. Crucially, DuckDB only + selects ``IE_JOIN`` for INNER pure-inequality joins — a bare ``SEMI JOIN`` / + ``ANTI JOIN`` inequality join is planned as a ``BLOCKWISE_NL_JOIN`` (a nested + loop, quadratic). So the left-only shapes are emitted as a correlated + ``WHERE EXISTS`` (SEMI) / ``WHERE NOT EXISTS`` (ANTI) subquery instead of a + semi/anti join keyword, which reaches ``IE_JOIN`` while preserving exact + semantics (#208). SEMI / ANTI restrict the outer SELECT to left-side + projections only. See :doc:`/transpilation/performance` for the full join-shape support matrix, fallback enumeration, and hard-error projection shapes. @@ -1028,45 +1036,73 @@ def _build_sql( l_table_ident = self._qualified_table_ident(sides.left_table) r_table_ident = self._qualified_table_ident(sides.right_table) - # Join keyword switches by ``sides.kind``: SEMI / ANTI engage - # DuckDB's IE_JOIN with the corresponding semi/anti semantics; INNER - # is the default. (CROSS folded to INNER inside ``_IEJoinSides``.) - join_keyword = { - "INNER": "JOIN", - "SEMI": "SEMI JOIN", - "ANTI": "ANTI JOIN", - }[sides.kind] - - inner_select_list = ", ".join(inner_projections) - per_chrom_select_prefix = f"SELECT {inner_select_list} " - from_left = f"FROM (SELECT * FROM {l_table_ident} WHERE {q(l_chrom)} = " - from_right_open = ( - f") a {join_keyword} (SELECT * FROM {r_table_ident} WHERE {q(r_chrom)} = " - ) - on_clause_predicates = [ + # The overlap predicate (canonical, strict) plus any inline residuals, + # shared by both the INNER join-ON form and the SEMI / ANTI EXISTS form. + overlap_predicates = [ f"{l_start_expr} < {r_end_expr}", f"{l_end_expr} > {r_start_expr}", ] for residual in inline_residuals: - on_clause_predicates.append( + overlap_predicates.append( self._rewrite_refs_for_per_chrom_subquery(residual, sides) ) - on_clause = ") b ON " + " AND ".join(on_clause_predicates) + predicate_sql = " AND ".join(overlap_predicates) # The dynamic SQL builder, expressed as a SQL string-concat that # ``string_agg`` aggregates per-chromosome. Single quotes are # doubled because the result is itself an SQL string literal that - # contains SQL. + # contains SQL. The chromosome literal is interpolated twice — once + # into the left partition filter, once into the right. chrom_literal = "'''' || replace(chrom, '''', '''''') || ''''" - esc = self._sql_escape - per_chrom_sql_expr = ( - f"'{esc(per_chrom_select_prefix)}{esc(from_left)}' " - f"|| {chrom_literal} " - f"|| '{esc(from_right_open)}' " - f"|| {chrom_literal} " - f"|| '{esc(on_clause)}'" - ) + inner_select_list = ", ".join(inner_projections) + + if sides.is_left_only_join: + # SEMI / ANTI: a bare per-chromosome ``SEMI JOIN`` / ``ANTI JOIN`` + # with an inequality overlap predicate is planned by DuckDB as a + # ``BLOCKWISE_NL_JOIN`` (a nested loop), not its fast ``IE_JOIN`` + # range join — DuckDB selects IE_JOIN only for INNER pure-inequality + # joins (#208). A correlated ``WHERE EXISTS`` / ``WHERE NOT EXISTS`` + # over the same per-chromosome right partition reaches IE_JOIN and + # preserves exact SEMI / ANTI semantics (one row per qualifying left + # row, no dedup-on-projection hazard). Inline (ON) residuals sit in + # the subquery WHERE beside the overlap predicate; WHERE residuals + # are still layered as an outer wrapper filter (``outer_where_residuals`` + # below, #200). + exists_keyword = "EXISTS" if sides.kind == "SEMI" else "NOT EXISTS" + select_from_left = ( + f"SELECT {inner_select_list} " + f"FROM (SELECT * FROM {l_table_ident} WHERE {q(l_chrom)} = " + ) + exists_open = ( + f") a WHERE {exists_keyword} (SELECT 1 FROM " + f"(SELECT * FROM {r_table_ident} WHERE {q(r_chrom)} = " + ) + exists_close = f") b WHERE {predicate_sql})" + per_chrom_sql_expr = ( + f"'{esc(select_from_left)}' " + f"|| {chrom_literal} " + f"|| '{esc(exists_open)}' " + f"|| {chrom_literal} " + f"|| '{esc(exists_close)}'" + ) + else: + # INNER (CROSS folded to INNER inside ``_IEJoinSides``): a + # per-chromosome INNER join whose pure-inequality ON predicate DuckDB + # plans through ``IE_JOIN``. + per_chrom_select_prefix = f"SELECT {inner_select_list} " + from_left = f"FROM (SELECT * FROM {l_table_ident} WHERE {q(l_chrom)} = " + from_right_open = ( + f") a JOIN (SELECT * FROM {r_table_ident} WHERE {q(r_chrom)} = " + ) + on_clause = f") b ON {predicate_sql}" + per_chrom_sql_expr = ( + f"'{esc(per_chrom_select_prefix)}{esc(from_left)}' " + f"|| {chrom_literal} " + f"|| '{esc(from_right_open)}' " + f"|| {chrom_literal} " + f"|| '{esc(on_clause)}'" + ) # Empty-schema fallback when no chromosomes pass the partition # filter. Project from the source tables under WHERE FALSE so diff --git a/tests/test_duckdb_iejoin.py b/tests/test_duckdb_iejoin.py index 3475396..e399f67 100644 --- a/tests/test_duckdb_iejoin.py +++ b/tests/test_duckdb_iejoin.py @@ -63,6 +63,22 @@ def _python_anti_overlap(peaks: list[tuple], genes: list[tuple]) -> list[tuple]: return sorted({(pc, ps, pe) for (pc, ps, pe) in peaks} - matched) +def _explain_dynamic_sql(conn, sql: str) -> str: + """Return the DuckDB EXPLAIN text of the per-chromosome dynamic SQL for *sql*. + + Runs the leading ``SET VARIABLE`` statement, reads the aggregated + per-chromosome ``UNION ALL`` string back out of the session variable, and + returns the plain EXPLAIN for it. The outer ``query(getvariable(...))`` + wrapper is opaque to EXPLAIN, so the dynamic SQL must be planned directly. + """ + statements = [s for s in sql.split(";\n") if s.strip()] + for statement in statements[:-1]: + conn.execute(statement) + var_name = re.search(r"__giql_iejoin_[0-9a-f]+", sql).group(0) + dynamic_sql = conn.execute(f"SELECT getvariable('{var_name}')").fetchone()[0] + return conn.execute("EXPLAIN " + dynamic_sql).fetchone()[1] + + @pytest.fixture def conn(): c = duckdb.connect(":memory:") @@ -369,8 +385,9 @@ def test_transpile_should_apply_anti_join_where_residual_as_outer_wrapper_filter The query is transpiled with ``dialect='duckdb'``. Then: The residual is applied as a ``WHERE`` on the outer wrapper - relation, never inlined into the per-chromosome ANTI ``ON`` (which - would invert the anti-join for rows failing the residual, #200). + relation, never inlined into the per-chromosome ``NOT EXISTS`` + subquery (which would invert the anti-join for rows failing the + residual, #200). """ # Arrange & act sql = transpile( @@ -384,9 +401,9 @@ def test_transpile_should_apply_anti_join_where_residual_as_outer_wrapper_filter # Assert per_chrom_template, _, outer_select = sql.partition("query(getvariable") assert "SET VARIABLE __giql_iejoin_" in sql - assert "ANTI JOIN" in per_chrom_template + assert "NOT EXISTS" in per_chrom_template # The residual predicate sits in the outer wrapper filter, never in the - # per-chromosome ANTI ON template that DuckDB expands via string_agg. + # per-chromosome NOT EXISTS template that DuckDB expands via string_agg. # Anchor on ``>= 550`` (the rendered comparison) rather than the bare # ``550`` literal: the per-chromosome template embeds the random # ``uuid4`` var-name token, whose hex could coincidentally contain the @@ -3034,16 +3051,18 @@ def test_transpile_should_engage_dialect_for_semi_join_with_intersects(self): # Assert assert "SET VARIABLE __giql_iejoin_" in sql - def test_transpile_should_emit_semi_keyword_when_join_is_semi(self): - """Test that SEMI JOIN's per-chromosome template uses the SEMI keyword. + def test_transpile_should_emit_exists_when_join_is_semi(self): + """Test that SEMI JOIN's per-chromosome template uses a correlated EXISTS. Given: A SEMI JOIN + INTERSECTS query. When: ``transpile`` is called with ``dialect='duckdb'``. Then: - The emitted SQL should contain a ``SEMI JOIN`` substring - inside the per-chromosome template (not just ``JOIN``). + The emitted SQL should express the semi-join as a correlated + ``WHERE EXISTS`` subquery (which DuckDB plans through IE_JOIN), + not a bare ``SEMI JOIN`` keyword (which DuckDB plans as a + nested loop, #208). """ # Arrange query = ( @@ -3055,7 +3074,34 @@ def test_transpile_should_emit_semi_keyword_when_join_is_semi(self): sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") # Assert - assert "SEMI JOIN" in sql + assert "EXISTS" in sql + assert "SEMI JOIN" not in sql + + def test_transpile_should_emit_not_exists_when_join_is_anti(self): + """Test that ANTI JOIN's per-chromosome template uses a correlated NOT EXISTS. + + Given: + An ANTI JOIN + INTERSECTS query. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The emitted SQL should express the anti-join as a correlated + ``WHERE NOT EXISTS`` subquery (which DuckDB plans through + IE_JOIN), not a bare ``ANTI JOIN`` keyword (which DuckDB plans + as a nested loop, #208). + """ + # Arrange + query = ( + "SELECT a.start FROM peaks a " + "ANTI JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Assert + assert "NOT EXISTS" in sql + assert "ANTI JOIN" not in sql def test_transpile_should_reject_right_side_projection_when_join_is_semi(self): """Test that SEMI JOIN raises on ``b.col`` in the outer SELECT. @@ -3155,6 +3201,80 @@ def test_transpile_should_reject_right_side_projection_when_join_is_anti(self): class TestTranspileDuckDBIEJoinExecution: """End-to-end execution tests against in-memory DuckDB.""" + def test_semi_join_plan_should_avoid_nested_loop_when_executed_on_duckdb(self, conn): + """Test that the SEMI-join EXISTS rewrite is not planned as a nested loop. + + Given: + A dense single-chromosome peaks/genes pair large enough that a + bare ``SEMI JOIN`` inequality overlap would be planned as a + quadratic ``BLOCKWISE_NL_JOIN``. + When: + The ``dialect='duckdb'`` SEMI-join SQL is generated and its + per-chromosome dynamic SQL is planned with ``EXPLAIN``. + Then: + It should avoid ``BLOCKWISE_NL_JOIN`` — the correlated + ``WHERE EXISTS`` form routes through DuckDB's range-join family + instead of a nested loop (#208). + """ + # Arrange + _make_table( + conn, "peaks", [("chr1", i * 7, i * 7 + 5, "p", 1, "+") for i in range(4000)] + ) + _make_table( + conn, + "genes", + [("chr1", i * 7 + 2, i * 7 + 9, "g", 1, "+") for i in range(4000)], + ) + sql = transpile( + "SELECT a.start FROM peaks a " + "SEMI JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + plan = _explain_dynamic_sql(conn, sql) + + # Assert + assert "BLOCKWISE_NL_JOIN" not in plan + + def test_anti_join_plan_should_avoid_nested_loop_when_executed_on_duckdb(self, conn): + """Test that the ANTI-join NOT EXISTS rewrite is not planned as a nested loop. + + Given: + A dense single-chromosome peaks/genes pair large enough that a + bare ``ANTI JOIN`` inequality overlap would be planned as a + quadratic ``BLOCKWISE_NL_JOIN``. + When: + The ``dialect='duckdb'`` ANTI-join SQL is generated and its + per-chromosome dynamic SQL is planned with ``EXPLAIN``. + Then: + It should avoid ``BLOCKWISE_NL_JOIN`` — the correlated + ``WHERE NOT EXISTS`` form routes through DuckDB's range-join + family instead of a nested loop (#208). + """ + # Arrange + _make_table( + conn, "peaks", [("chr1", i * 7, i * 7 + 5, "p", 1, "+") for i in range(4000)] + ) + _make_table( + conn, + "genes", + [("chr1", i * 7 + 2, i * 7 + 9, "g", 1, "+") for i in range(4000)], + ) + sql = transpile( + "SELECT a.start FROM peaks a " + "ANTI JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + plan = _explain_dynamic_sql(conn, sql) + + # Assert + assert "BLOCKWISE_NL_JOIN" not in plan + def test_query_should_return_overlapping_pairs_when_executed_on_duckdb( self, peaks_genes ): @@ -5489,20 +5609,19 @@ def test_query_should_match_python_native_semi_overlap_for_random_inputs( max_size=6, ), ) - def test_query_should_match_naive_predicate_plan_for_semi_join_random_inputs( + def test_query_should_match_python_native_semi_overlap_for_narrow_random_inputs( self, conn, peaks, genes ): - """Test that SEMI JOIN under the dialect matches the naive-predicate plan. + """Test SEMI JOIN under the dialect against a Python-native reference. Given: A Hypothesis-generated pair of small interval lists. When: - The same SEMI JOIN INTERSECTS query is transpiled with - ``dialect=None`` (naive predicate) and ``dialect='duckdb'`` and - executed. + A ``SEMI JOIN ... ON a.interval INTERSECTS b.interval`` is + transpiled with ``dialect='duckdb'`` and executed. Then: - Both plans should return the same multiset of distinct - left rows. + The set of returned rows should equal the Python reference of + distinct left rows with at least one overlapping right row. """ # Arrange peak_rows = [(c, s, s + length) for (c, s, length) in peaks] @@ -5524,15 +5643,14 @@ def test_query_should_match_naive_predicate_plan_for_semi_join_random_inputs( "FROM peaks a " "SEMI JOIN genes b ON a.interval INTERSECTS b.interval" ) - sql_default = transpile(query, tables=["peaks", "genes"]) sql_duckdb = transpile(query, tables=["peaks", "genes"], dialect="duckdb") # Act - rows_default = sorted(set(conn.execute(sql_default).fetchall())) rows_duckdb = sorted(set(conn.execute(sql_duckdb).fetchall())) + expected = _python_semi_overlap(peak_rows, gene_rows) # Assert - assert rows_default == rows_duckdb + assert rows_duckdb == expected @settings( max_examples=25, @@ -5630,22 +5748,21 @@ def test_query_should_match_python_native_anti_overlap_for_random_inputs( max_size=6, ), ) - def test_query_should_match_naive_predicate_plan_for_anti_join_random_inputs( + def test_query_should_match_python_native_anti_overlap_for_narrow_random_inputs( self, conn, peaks, genes ): - """Test that ANTI JOIN under the dialect matches the naive-predicate plan. + """Test ANTI JOIN under the dialect against a Python-native reference. Given: A Hypothesis-generated pair of small interval lists. When: - The same ANTI JOIN INTERSECTS query is transpiled with - ``dialect=None`` (naive predicate) and ``dialect='duckdb'`` - and executed. + An ``ANTI JOIN ... ON a.interval INTERSECTS b.interval`` is + transpiled with ``dialect='duckdb'`` and executed. Then: - Both plans should return the same multiset of distinct left - rows, including left rows on chromosomes absent from the - right table and every left row when the right table is empty - (standard ``ANTI JOIN ... ON `` semantics). + The set of returned rows should equal the Python reference of + distinct left rows with no overlapping right row, including + rows on chromosomes absent from the right table and every left + row when the right table is empty. """ # Arrange peak_rows = [(c, s, s + length) for (c, s, length) in peaks] @@ -5667,15 +5784,14 @@ def test_query_should_match_naive_predicate_plan_for_anti_join_random_inputs( "FROM peaks a " "ANTI JOIN genes b ON a.interval INTERSECTS b.interval" ) - sql_default = transpile(query, tables=["peaks", "genes"]) sql_duckdb = transpile(query, tables=["peaks", "genes"], dialect="duckdb") # Act - rows_default = sorted(set(conn.execute(sql_default).fetchall())) rows_duckdb = sorted(set(conn.execute(sql_duckdb).fetchall())) + expected = _python_anti_overlap(peak_rows, gene_rows) # Assert - assert rows_default == rows_duckdb + assert rows_duckdb == expected def test_query_should_match_naive_plan_for_anti_join_with_where_residual(self, conn): """Test that an ANTI-join WHERE residual filters without inverting the join. @@ -5841,21 +5957,21 @@ def test_query_should_match_naive_plan_for_semi_join_with_where_residual(self, c max_size=6, ), ) - def test_query_should_match_naive_plan_for_anti_join_where_residual_random_inputs( + def test_query_should_match_python_native_anti_overlap_with_where_residual_random_inputs( self, conn, peaks, genes ): - """Test ANTI JOIN with a WHERE residual against the naive plan over random inputs. + """Test ANTI JOIN with a WHERE residual against a Python-native reference. Given: A Hypothesis-generated pair of small interval lists. When: An ``ANTI JOIN ... ON INTERSECTS ... WHERE a.start >= 50`` is - transpiled with ``dialect=None`` and ``dialect='duckdb'`` and - executed. + transpiled with ``dialect='duckdb'`` and executed. Then: - Both plans return the same rows for every input — the WHERE - filter never resurrects an anti-excluded row on either side of - the 50 threshold (#200). + The set of returned rows should equal the Python anti-overlap + reference filtered to ``start >= 50`` — the WHERE filter never + resurrects an anti-excluded row on either side of the 50 + threshold (#200). """ # Arrange peak_rows = [(c, s, s + length) for (c, s, length) in peaks] @@ -5878,15 +5994,16 @@ def test_query_should_match_naive_plan_for_anti_join_where_residual_random_input "ANTI JOIN genes b ON a.interval INTERSECTS b.interval " "WHERE a.start >= 50" ) - sql_default = transpile(query, tables=["peaks", "genes"]) sql_duckdb = transpile(query, tables=["peaks", "genes"], dialect="duckdb") # Act - rows_default = sorted(set(conn.execute(sql_default).fetchall())) rows_duckdb = sorted(set(conn.execute(sql_duckdb).fetchall())) + expected = sorted( + row for row in _python_anti_overlap(peak_rows, gene_rows) if row[1] >= 50 + ) # Assert - assert rows_default == rows_duckdb + assert rows_duckdb == expected def test_query_should_inline_anti_join_on_residual_referencing_right_side( self, conn